"""Face detection and processing services for PunimTag Web."""

from __future__ import annotations

import json
import os
import tempfile
import time
from typing import Callable, Optional, Tuple, List, Dict
from datetime import date

import numpy as np
from PIL import Image
from sqlalchemy.orm import Session, joinedload
from sqlalchemy import and_, func, case

# Skip DeepFace import during tests to avoid illegal instruction errors
if os.getenv("SKIP_DEEPFACE_IN_TESTS") == "1":
    DEEPFACE_AVAILABLE = False
    DeepFace = None
else:
    try:
        from deepface import DeepFace
        DEEPFACE_AVAILABLE = True
    except ImportError:
        DEEPFACE_AVAILABLE = False
        DeepFace = None

from backend.config import (
    CONFIDENCE_CALIBRATION_METHOD,
    DEFAULT_FACE_TOLERANCE,
    DEEPFACE_ALIGN_FACES,
    DEEPFACE_ENFORCE_DETECTION,
    MAX_FACE_SIZE,
    MIN_FACE_CONFIDENCE,
    MIN_FACE_SIZE,
    USE_CALIBRATED_CONFIDENCE,
)
from src.utils.exif_utils import EXIFOrientationHandler
from src.utils.pose_detection import PoseDetector, RETINAFACE_AVAILABLE
from backend.db.models import Face, Photo, Person, Tag, PhotoTagLinkage


def _pre_warm_deepface(
    detector_backend: str,
    model_name: str,
    update_progress: Optional[Callable[[int, int, str, int, int], None]] = None,
) -> None:
    """Pre-warm DeepFace models by making a small initialization call.
    
    This triggers model loading/downloading before processing actual photos,
    so the delay happens during initialization progress, not during first photo.
    
    Args:
        detector_backend: DeepFace detector backend
        model_name: DeepFace model name
        update_progress: Optional progress callback
    """
    if not DEEPFACE_AVAILABLE:
        return
    
    if update_progress:
        update_progress(0, 0, "Loading DeepFace models (this may take a moment)...", 0, 0)
    
    print(f"[DeepFace] Pre-warming models: detector={detector_backend}, model={model_name}")
    start_time = time.time()
    
    try:
        # Create a minimal test image to trigger model loading
        # This forces DeepFace to download/load models without processing a real photo
        # Using a small grayscale image (100x100) is sufficient to trigger initialization
        dummy_img = Image.new('RGB', (100, 100), color='black')
        dummy_array = np.array(dummy_img)
        
        # Suppress stderr to prevent broken pipe errors from gdown
        import sys
        import contextlib
        from io import StringIO
        
        # Suppress stdout/stderr during model loading
        with contextlib.redirect_stdout(StringIO()), open(os.devnull, 'w') as devnull:
            old_stderr = sys.stderr
            sys.stderr = devnull
            try:
                # This will trigger model loading but won't find any faces (which is fine)
                # We use enforce_detection=False to avoid errors when no faces are found
                DeepFace.represent(
                    img_path=dummy_array,
                    model_name=model_name,
                    detector_backend=detector_backend,
                    enforce_detection=False,  # Don't fail if no faces
                    align=DEEPFACE_ALIGN_FACES,
                )
            finally:
                sys.stderr = old_stderr
        
        elapsed = time.time() - start_time
        print(f"[DeepFace] Models loaded in {elapsed:.2f}s")
        if update_progress:
            update_progress(0, 0, f"DeepFace models ready ({elapsed:.1f}s)", 0, 0)
    except Exception as e:
        # If pre-warming fails, models will just load on first real photo
        elapsed = time.time() - start_time
        try:
            print(f"[DeepFace] Pre-warming completed (with warnings) in {elapsed:.2f}s: {e}")
        except (BrokenPipeError, OSError):
            pass
        # Don't raise - let it load on first photo instead


def calculate_face_quality_score(
    image_np: np.ndarray,
    face_location: dict,
    image_width: int,
    image_height: int,
) -> int:
    """Calculate face quality score (0-100).
    
    This matches the desktop version logic exactly from src/core/face_processing.py _calculate_face_quality_score()
    Returns 0-100 (will be converted to 0.0-1.0 for database storage).
    
    Args:
        image_np: Image as numpy array
        face_location: Face location dict with x, y, w, h
        image_width: Image width
        image_height: Image height
        
    Returns:
        Quality score from 0-100
    """
    try:
        # DeepFace format: {x, y, w, h}
        x = face_location.get('x', 0)
        y = face_location.get('y', 0)
        w = face_location.get('w', 0)
        h = face_location.get('h', 0)
        
        face_height = h
        face_width = w
        left = x
        right = x + w
        top = y
        bottom = y + h
        
        # Basic size check - faces too small get lower scores
        min_face_size = 50
        size_score = min(1.0, (face_height * face_width) / (min_face_size * min_face_size))
        
        # Extract face region
        face_region = image_np[top:bottom, left:right]
        if face_region.size == 0:
            return 0
        
        # Convert to grayscale for analysis
        if len(face_region.shape) == 3:
            gray_face = np.mean(face_region, axis=2)
        else:
            gray_face = face_region
        
        # Calculate sharpness (Laplacian variance)
        # Match desktop version exactly (including the bug for consistency)
        # Desktop calculates var of kernel array itself, not the convolved result
        laplacian_var = np.var(np.array([[0, -1, 0], [-1, 4, -1], [0, -1, 0]]).astype(np.float32))
        if laplacian_var > 0:
            sharpness = np.var(np.array([[0, -1, 0], [-1, 4, -1], [0, -1, 0]]).astype(np.float32))
        else:
            sharpness = 0.0
        sharpness_score = min(1.0, sharpness / 1000.0)  # Normalize sharpness
        
        # Calculate brightness and contrast
        mean_brightness = np.mean(gray_face)
        brightness_score = 1.0 - abs(mean_brightness - 128) / 128.0  # Prefer middle brightness
        
        contrast = np.std(gray_face)
        contrast_score = min(1.0, contrast / 64.0)  # Prefer good contrast
        
        # Calculate aspect ratio (faces should be roughly square)
        aspect_ratio = face_width / face_height if face_height > 0 else 1.0
        aspect_score = 1.0 - abs(aspect_ratio - 1.0)  # Prefer square faces
        
        # Calculate position in image (centered faces are better)
        img_height, img_width = image_np.shape[:2]
        center_x = (left + right) / 2
        center_y = (top + bottom) / 2
        position_x_score = 1.0 - abs(center_x - img_width / 2) / (img_width / 2)
        position_y_score = 1.0 - abs(center_y - img_height / 2) / (img_height / 2)
        position_score = (position_x_score + position_y_score) / 2.0
        
        # Weighted combination of all factors (matches desktop exactly)
        quality_score = (
            size_score * 0.25 +
            sharpness_score * 0.25 +
            brightness_score * 0.15 +
            contrast_score * 0.15 +
            aspect_score * 0.10 +
            position_score * 0.10
        )
        
        # Desktop returns 0.0-1.0, we need 0-100 for database
        quality_score = max(0.0, min(1.0, quality_score))
        return int(quality_score * 100)
        
    except Exception as e:
        print(f"[FaceService] ⚠️  Error calculating face quality score: {e}")
        # Return a default quality score on error
        return 50


def is_valid_face_detection(
    confidence: float,
    face_location: dict,
    image_width: int,
    image_height: int,
) -> bool:
    """Check if face detection meets minimum criteria.
    
    Args:
        confidence: Face detection confidence score (0-1 range)
        face_location: Face location dict with x, y, w, h
        image_width: Image width
        image_height: Image height
        
    Returns:
        True if face is valid, False otherwise
    """
    is_valid, _ = is_valid_face_detection_with_reason(
        confidence, face_location, image_width, image_height
    )
    return is_valid


def is_valid_face_detection_with_reason(
    confidence: float,
    face_location: dict,
    image_width: int,
    image_height: int,
) -> Tuple[bool, str]:
    """Check if face detection meets minimum criteria and return reason if invalid.
    
    This matches the desktop version logic EXACTLY from src/core/face_processing.py _is_valid_face_detection()
    
    Args:
        confidence: Face detection confidence score (0-1 range from DeepFace)
        face_location: Face location dict with x, y, w, h
        image_width: Image width (unused but kept for compatibility)
        image_height: Image height (unused but kept for compatibility)
        
    Returns:
        Tuple of (is_valid: bool, reason: str)
        reason is empty string if valid, otherwise describes why it failed
    """
    try:
        # Desktop version uses face_confidence directly (0-1 range from DeepFace)
        # No normalization needed if DeepFace returns 0-1 range
        face_confidence = confidence
        
        # Check confidence threshold - be more strict
        if face_confidence < MIN_FACE_CONFIDENCE:
            return False, f"confidence too low (got {face_confidence:.3f}, need >= {MIN_FACE_CONFIDENCE})"
        
        # Check face size
        width = face_location.get('w', 0)
        height = face_location.get('h', 0)
        
        # Too small faces are likely false positives (balloons, decorations, etc.)
        if width < MIN_FACE_SIZE or height < MIN_FACE_SIZE:
            return False, f"size too small (got {width}x{height}, need >= {MIN_FACE_SIZE}x{MIN_FACE_SIZE})"
        
        # Too large faces might be full-image false positives
        if width > MAX_FACE_SIZE or height > MAX_FACE_SIZE:
            return False, f"size too large (got {width}x{height}, need <= {MAX_FACE_SIZE}x{MAX_FACE_SIZE})"
        
        # Check aspect ratio - faces should be roughly square (not too wide/tall)
        aspect_ratio = width / height if height > 0 else 1.0
        if aspect_ratio < 0.4 or aspect_ratio > 2.5:  # More strict aspect ratio (was 0.3-3.0)
            return False, f"aspect ratio out of range (got {aspect_ratio:.2f}, need 0.4-2.5)"
        
        # Additional filtering for very small faces with low confidence
        # Small faces need higher confidence to be accepted
        face_area = width * height
        if face_area < 6400:  # Less than 80x80 pixels (lowered from 100x100)
            if face_confidence < 0.6:  # Require 60% confidence for small faces (lowered from 80%)
                return False, f"small face needs higher confidence (area={face_area}, confidence={face_confidence:.3f}, need >= 0.6)"
        
        # Filter out faces that are too close to image edges (often false positives)
        x = face_location.get('x', 0)
        y = face_location.get('y', 0)
        # If face is very close to edges, require higher confidence
        if x < 10 or y < 10:  # Within 10 pixels of top/left edge
            if face_confidence < 0.65:  # Require 65% confidence for edge faces (lowered from 85%)
                return False, f"edge face needs higher confidence (x={x}, y={y}, confidence={face_confidence:.3f}, need >= 0.65)"
        
        return True, ""
        
    except Exception as e:
        # Desktop version defaults to accepting on error
        print(f"[FaceService] ⚠️  Error validating face detection: {e}")
        return True, ""


def process_photo_faces(
    db: Session,
    photo: Photo,
    detector_backend: str = "retinaface",
    model_name: str = "ArcFace",
    update_progress: Optional[Callable[[int, int, str], None]] = None,
    pose_detector: Optional[PoseDetector] = None,
) -> Tuple[int, int]:
    """Process faces in a single photo using DeepFace.
    
    Args:
        db: Database session
        photo: Photo model instance
        detector_backend: DeepFace detector backend (retinaface, mtcnn, opencv, ssd)
        model_name: DeepFace model name (ArcFace, Facenet, Facenet512, VGG-Face)
        update_progress: Optional progress callback (processed, total, message)
        pose_detector: Optional PoseDetector instance to reuse (initialized once per batch)
                      If None and RETINAFACE_AVAILABLE, will create one locally
        
    Returns:
        Tuple of (faces_detected, faces_stored)
    """
    if not DEEPFACE_AVAILABLE:
        raise RuntimeError("DeepFace not available")
    
    # Suppress stderr globally for this function to prevent broken pipe errors
    # This must happen BEFORE any DeepFace/RetinaFace initialization
    import sys
    from io import StringIO
    import contextlib
    
    # Save original stderr and redirect to devnull for the entire function
    # This prevents gdown (used by RetinaFace/DeepFace) from causing broken pipe errors
    # Note: os is imported at module level, so it's available here
    original_stderr = sys.stderr
    devnull_path = os.devnull  # Capture os.devnull value to avoid closure issues
    devnull_fd = open(devnull_path, 'w')
    sys.stderr = devnull_fd
    
    def _print_with_stderr(*args, **kwargs):
        """Helper to temporarily restore stderr for print statements"""
        sys.stderr = original_stderr
        try:
            print(*args, **kwargs)
        finally:
            sys.stderr = devnull_fd
    
    try:
        
        photo_path = photo.path
        if not os.path.exists(photo_path):
            sys.stderr = original_stderr
            if devnull_fd:
                devnull_fd.close()
            return 0, 0
        
        # Skip videos (videos are not processed for face detection)
        try:
            media_type = getattr(photo, 'media_type', 'image')
            if media_type == 'video':
                sys.stderr = original_stderr
                if devnull_fd:
                    devnull_fd.close()
                return 0, 0
        except Exception:
            pass
        
        # Skip if already processed (desktop parity)
        try:
            if getattr(photo, 'processed', False):
                sys.stderr = original_stderr
                if devnull_fd:
                    devnull_fd.close()
                return 0, 0
        except Exception:
            pass
        
        # Get EXIF orientation
        exif_orientation = EXIFOrientationHandler.get_exif_orientation(photo_path)
        
        # Apply EXIF orientation correction
        corrected_image, original_orientation = (
            EXIFOrientationHandler.correct_image_orientation_from_path(photo_path)
        )
        
        temp_path = None
        if corrected_image is not None and original_orientation and original_orientation != 1:
            # Save corrected image temporarily
            temp_dir = tempfile.gettempdir()
            temp_filename = f"corrected_{photo.id}_{os.path.basename(photo_path)}"
            temp_path = os.path.join(temp_dir, temp_filename)
            corrected_image.save(temp_path, "JPEG", quality=95)
            face_detection_path = temp_path
        else:
            face_detection_path = photo_path
        
        # Step 1: Use RetinaFace directly for detection + landmarks (with graceful fallback)
        # Reuse the pose_detector passed in (initialized once per batch) or create one if needed
        pose_faces = []
        if pose_detector is not None:
            # Use the shared detector instance (much faster - no reinitialization)
            try:
                pose_faces = pose_detector.detect_pose_faces(face_detection_path)
                if pose_faces:
                    _print_with_stderr(f"[FaceService] Pose detection for {photo.filename}: found {len(pose_faces)} faces with pose data")
            except Exception as e:
                _print_with_stderr(f"[FaceService] ⚠️  Pose detection failed for {photo.filename}: {e}, using defaults")
                pose_faces = []
        elif RETINAFACE_AVAILABLE:
            # Fallback: create detector if not provided (backward compatibility)
            # Note: stderr is already suppressed above, so PoseDetector initialization won't cause broken pipe
            try:
                pose_detector_local = PoseDetector()
                pose_faces = pose_detector_local.detect_pose_faces(face_detection_path)
                if pose_faces:
                    _print_with_stderr(f"[FaceService] Pose detection for {photo.filename}: found {len(pose_faces)} faces with pose data")
            except Exception as e:
                _print_with_stderr(f"[FaceService] ⚠️  Pose detection failed for {photo.filename}: {e}, using defaults")
                pose_faces = []
        
        # Step 2: Use DeepFace for encoding generation
        # Note: stderr is already suppressed, so DeepFace won't cause broken pipe errors
        _print_with_stderr(f"[DeepFace] Processing {photo.filename} with {detector_backend}/{model_name}...")
        
        # Check if model weights exist before calling DeepFace
        # This prevents automatic download attempts that can cause broken pipe errors
        weights_path = os.path.expanduser(f"~/.deepface/weights/arcface_weights.h5")
        if model_name == "ArcFace" and not os.path.exists(weights_path):
            sys.stderr = original_stderr
            raise Exception(
                f"ArcFace model weights not found at {weights_path}. "
                f"Please download manually: "
                f"wget https://github.com/serengil/deepface_models/releases/download/v1.0/arcface_weights.h5 "
                f"-O {weights_path}"
            )
        
        # DeepFace call - stderr is already suppressed
        try:
            results = DeepFace.represent(
                img_path=face_detection_path,
                model_name=model_name,
                detector_backend=detector_backend,
                enforce_detection=DEEPFACE_ENFORCE_DETECTION,
                align=DEEPFACE_ALIGN_FACES,
            )
            _print_with_stderr(f"[DeepFace] Completed {photo.filename}")
        except Exception as e:
            error_msg = str(e)
            try:
                _print_with_stderr(f"[DeepFace] Error processing {photo.filename}: {error_msg}")
            except (BrokenPipeError, OSError):
                pass  # Ignore broken pipe when printing
            
            # If it's a model download error, provide helpful message
            if "downloading" in error_msg.lower() or "arcface_weights" in error_msg.lower():
                raise Exception(
                    f"Failed to load DeepFace model '{model_name}'. "
                    f"Please download the model weights manually. Error: {error_msg}"
                )
            raise
        finally:
            # Clean up temporary file if created
            if temp_path and os.path.exists(temp_path):
                try:
                    os.remove(temp_path)
                except Exception:
                    pass
        
        if not results:
            # Mark photo as processed even if no faces found (desktop parity)
            try:
                photo.processed = True
                db.add(photo)
                db.commit()
            except Exception:
                db.rollback()
            return 0, 0
        
        # Load image for quality calculation
        image = Image.open(photo_path)
        image_np = np.array(image)
        image_width, image_height = image.size
        
        # Count total faces from DeepFace
        faces_detected = len(results)
        faces_stored = 0
        validation_failures = {}  # Track failures by reason type
        
        _print_with_stderr(f"[FaceService] Processing {faces_detected} faces from DeepFace for {photo.filename} "
              f"(image size: {image_width}x{image_height})")
        
        # Track which pose_faces have been used to prevent duplicate matches
        used_pose_indices = set()
        
        for idx, result in enumerate(results):
            # Debug: Print full result to see what DeepFace returns
            if idx == 0:
                _print_with_stderr(f"[FaceService] Debug - DeepFace result keys: {result.keys()}")
                _print_with_stderr(f"[FaceService] Debug - Sample result structure: {list(result.keys())}")
            
            facial_area = result.get('facial_area', {})
            # Try multiple possible confidence fields (matching desktop version)
            # Desktop uses: result.get('face_confidence', 0.0)
            face_confidence = result.get('face_confidence', 0.0)
            
            # If confidence is 0.0, DeepFace might not provide it for this detector
            # Some detectors don't return confidence - in that case, use a default
            # Default to 0.5 (medium confidence) if missing, so faces aren't automatically rejected
            if face_confidence == 0.0:
                # Try alternative fields
                face_confidence = result.get('confidence', 
                                  result.get('detection_confidence', 
                                  facial_area.get('confidence', 0.5)))  # Default to 0.5 if completely missing
            
            # Debug first face to see what DeepFace returns
            if idx == 0:
                _print_with_stderr(f"[FaceService] Debug - DeepFace result keys: {list(result.keys())}")
                if facial_area:
                    _print_with_stderr(f"[FaceService] Debug - facial_area keys: {list(facial_area.keys())}")
                    _print_with_stderr(f"[FaceService] Debug - facial_area content: {facial_area}")
                _print_with_stderr(f"[FaceService] Debug - face_confidence value: {face_confidence}")
                _print_with_stderr(f"[FaceService] Debug - result['face_confidence'] exists: {'face_confidence' in result}")
            
            encoding = np.array(result['embedding'])
            
            # Convert to location format (JSON string like desktop version)
            location = {
                'x': facial_area.get('x', 0),
                'y': facial_area.get('y', 0),
                'w': facial_area.get('w', 0),
                'h': facial_area.get('h', 0),
            }
            location_str = json.dumps(location)
            
            # Validate face detection with detailed error reporting
            # Match desktop version: pass confidence as-is, validation function handles normalization
            is_valid, reason = is_valid_face_detection_with_reason(
                face_confidence, location, image_width, image_height
            )
            
            if not is_valid:
                # Extract failure type from reason (first word before colon)
                failure_type = reason.split(':')[0].strip() if ':' in reason else reason
                validation_failures[failure_type] = validation_failures.get(failure_type, 0) + 1
                _print_with_stderr(f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename} failed validation: "
                      f"{reason} - confidence={face_confidence:.3f}, "
                      f"location={location}, size={location['w']}x{location['h']}")
                continue
            
            _print_with_stderr(f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename} passed validation: "
                  f"confidence={face_confidence:.3f}, size={location['w']}x{location['h']}")
            
            # Calculate quality score (0.0-1.0 range for desktop compatibility)
            quality_score_int = calculate_face_quality_score(
                image_np, location, image_width, image_height
            )
            # Convert from 0-100 to 0.0-1.0 for database (desktop stores REAL)
            quality_score = quality_score_int / 100.0
            
            # Step 3: Match RetinaFace results with DeepFace results
            # Returns (pose_info, matched_pose_face_index) to allow tracking used poses
            # Pass used_pose_indices to exclude already-matched poses
            pose_info, matched_index = _find_matching_pose_info(
                facial_area, pose_faces, used_pose_indices
            )
            yaw_angle = pose_info.get('yaw_angle')
            pitch_angle = pose_info.get('pitch_angle')
            roll_angle = pose_info.get('roll_angle')
            face_width = pose_info.get('face_width')  # Extract face width for verification
            
            # Recalculate pose_mode using classify_pose_mode if we have face_width but no yaw
            # This ensures profile faces are detected even when yaw calculation fails
            from src.utils.pose_detection import PoseDetector
            if yaw_angle is None:
                # Try to get yaw from the matched pose_face if available
                # This helps determine direction (left vs right) when yaw calculation failed
                matched_pose_face = None
                if matched_index is not None and matched_index < len(pose_faces):
                    matched_pose_face = pose_faces[matched_index]
                elif pose_faces:
                    # Try to find a pose_face by location (best match by center distance)
                    # This handles cases where initial matching failed but we still have pose data
                    face_center_x = facial_area.get('x', 0) + facial_area.get('w', 0) / 2
                    face_center_y = facial_area.get('y', 0) + facial_area.get('h', 0) / 2
                    best_distance = float('inf')
                    
                    for pf in pose_faces:
                        if not pf.get('landmarks'):
                            continue
                        
                        pose_area = pf.get('facial_area', {})
                        if isinstance(pose_area, dict) and pose_area:
                            pose_center_x = pose_area.get('x', 0) + pose_area.get('w', 0) / 2
                            pose_center_y = pose_area.get('y', 0) + pose_area.get('h', 0) / 2
                            distance = ((face_center_x - pose_center_x) ** 2 + 
                                       (face_center_y - pose_center_y) ** 2) ** 0.5
                            
                            if distance < best_distance:
                                best_distance = distance
                                matched_pose_face = pf
                    
                    # If location matching failed, try matching by face_width as fallback
                    if matched_pose_face is None and face_width is not None:
                        for pf in pose_faces:
                            if pf.get('landmarks') and pf.get('face_width') == face_width:
                                matched_pose_face = pf
                                break
                
                # If we have landmarks, try to calculate yaw to determine direction
                if matched_pose_face and matched_pose_face.get('landmarks'):
                    landmarks = matched_pose_face.get('landmarks')
                    calculated_yaw = PoseDetector.calculate_yaw_from_landmarks(landmarks)
                    if calculated_yaw is not None:
                        # Use calculated yaw for classification
                        yaw_angle = calculated_yaw
                        # Also get pitch and roll if available
                        if pitch_angle is None:
                            pitch_angle = matched_pose_face.get('pitch_angle')
                        if roll_angle is None:
                            roll_angle = matched_pose_face.get('roll_angle')
                        # Update face_width if we have it from matched pose
                        if face_width is None:
                            face_width = matched_pose_face.get('face_width')
                        pose_mode = PoseDetector.classify_pose_mode(
                            yaw_angle, pitch_angle, roll_angle, face_width
                        )
                    else:
                        # Can't calculate yaw, use face_width
                        pose_mode = PoseDetector.classify_pose_mode(
                            yaw_angle, pitch_angle, roll_angle, face_width
                        )
                elif face_width is not None:
                    # No landmarks available, use face_width only
                    pose_mode = PoseDetector.classify_pose_mode(
                        yaw_angle, pitch_angle, roll_angle, face_width
                    )
                else:
                    # No landmarks and no face_width, use default
                    pose_mode = pose_info.get('pose_mode', 'frontal')
            else:
                # Use the pose_mode from matching (or default to frontal)
                pose_mode = pose_info.get('pose_mode', 'frontal')
            
            # Mark matched pose_face as used to prevent re-matching to other faces
            if matched_index is not None:
                used_pose_indices.add(matched_index)
            
            # Get landmarks from matched pose_face for storage
            landmarks_json = None
            matched_pose_face = None
            if matched_index is not None and matched_index < len(pose_faces):
                matched_pose_face = pose_faces[matched_index]
            elif pose_faces:
                # Try to find a pose_face by location (best match by center distance)
                face_center_x = facial_area.get('x', 0) + facial_area.get('w', 0) / 2
                face_center_y = facial_area.get('y', 0) + facial_area.get('h', 0) / 2
                best_distance = float('inf')
                
                for pf in pose_faces:
                    if not pf.get('landmarks'):
                        continue
                    
                    pose_area = pf.get('facial_area', {})
                    if isinstance(pose_area, dict) and pose_area:
                        pose_center_x = pose_area.get('x', 0) + pose_area.get('w', 0) / 2
                        pose_center_y = pose_area.get('y', 0) + pose_area.get('h', 0) / 2
                        distance = ((face_center_x - pose_center_x) ** 2 + 
                                   (face_center_y - pose_center_y) ** 2) ** 0.5
                        
                        if distance < best_distance:
                            best_distance = distance
                            matched_pose_face = pf
            
            # Convert landmarks dict to JSON string if available
            if matched_pose_face and matched_pose_face.get('landmarks'):
                landmarks_dict = matched_pose_face.get('landmarks')
                # Convert tuple coordinates to lists and numpy types to native Python types for JSON serialization
                landmarks_serializable = {}
                for key, value in landmarks_dict.items():
                    if isinstance(value, (tuple, list)):
                        # Convert tuple/list and handle numpy types within
                        landmarks_serializable[key] = [
                            float(v) if isinstance(v, np.floating) else int(v) if isinstance(v, np.integer) else v
                            for v in value
                        ]
                    elif isinstance(value, np.floating):
                        # Convert numpy float types to native Python float
                        landmarks_serializable[key] = float(value)
                    elif isinstance(value, np.integer):
                        # Convert numpy integer types to native Python int
                        landmarks_serializable[key] = int(value)
                    else:
                        landmarks_serializable[key] = value
                landmarks_json = json.dumps(landmarks_serializable)
            
            # Log face width for profile detection verification
            if face_width is not None:
                profile_status = "PROFILE" if face_width < 25.0 else "FRONTAL"
                yaw_str = f"{yaw_angle:.2f}°" if yaw_angle is not None else "None"
                _print_with_stderr(f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename}: "
                      f"face_width={face_width:.2f}px, pose_mode={pose_mode} ({profile_status}), yaw={yaw_str}")
            else:
                yaw_str = f"{yaw_angle:.2f}°" if yaw_angle is not None else "None"
                _print_with_stderr(f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename}: "
                      f"face_width=None, pose_mode={pose_mode}, yaw={yaw_str}")
            
            # Store face in database - match desktop schema exactly
            # Desktop: confidence REAL DEFAULT 0.0 (legacy), face_confidence REAL (actual)
            # Desktop: quality_score REAL DEFAULT 0.0 (0.0-1.0 range)
            face = Face(
                photo_id=photo.id,
                person_id=None,
                encoding=encoding.tobytes(),
                location=location_str,
                confidence=0.0,  # Legacy field (desktop keeps at 0.0)
                quality_score=quality_score,  # REAL in 0.0-1.0 range
                is_primary_encoding=False,
                detector_backend=detector_backend,
                model_name=model_name,
                face_confidence=face_confidence,  # REAL in 0.0-1.0 range
                exif_orientation=exif_orientation,
                pose_mode=pose_mode,
                yaw_angle=yaw_angle,
                pitch_angle=pitch_angle,
                roll_angle=roll_angle,
                landmarks=landmarks_json,
            )
            
            db.add(face)
            faces_stored += 1
        
        # Commit all faces at once - wrap in try-except to handle database errors gracefully
        try:
            db.commit()
        except Exception as commit_error:
            # If commit fails, rollback and log the error
            db.rollback()
            error_msg = str(commit_error)
            try:
                _print_with_stderr(f"[FaceService] Failed to commit {faces_stored} faces for {photo.filename}: {error_msg}")
                import traceback
                traceback.print_exc()
            except (BrokenPipeError, OSError):
                pass
            
            # Reset faces_stored since commit failed - faces weren't actually stored
            # This ensures the return value accurately reflects what was actually saved
            faces_stored = 0
            
            # Re-raise to be caught by outer exception handler in process_unprocessed_photos
            # This allows the batch to continue processing other photos
            raise Exception(f"Database commit failed for {photo.filename}: {error_msg}")
        
        # Mark photo as processed after handling faces (desktop parity)
        try:
            photo.processed = True
            db.add(photo)
            db.commit()
        except Exception:
            db.rollback()
        
        # Log summary
        if faces_stored < faces_detected:
            _print_with_stderr(f"[FaceService] Summary for {photo.filename}: "
                  f"{faces_detected} faces detected, {faces_stored} faces stored, "
                  f"{faces_detected - faces_stored} failed validation")
            for reason, count in validation_failures.items():
                if count > 0:
                    _print_with_stderr(f"[FaceService]   - {reason}: {count}")
        else:
            _print_with_stderr(f"[FaceService] Summary for {photo.filename}: "
                  f"{faces_detected} faces detected, {faces_stored} faces stored")
        
        return faces_detected, faces_stored
        
    except Exception as e:
        db.rollback()
        raise Exception(f"Error processing faces in {photo.filename}: {str(e)}")
    finally:
        # Always restore stderr
        sys.stderr = original_stderr
        if devnull_fd:
            try:
                devnull_fd.close()
            except Exception:
                pass


def _calculate_iou(box1: Dict, box2: Dict) -> float:
    """Calculate Intersection over Union (IoU) between two bounding boxes.
    
    Args:
        box1: First bounding box {'x': x, 'y': y, 'w': w, 'h': h}
        box2: Second bounding box {'x': x, 'y': y, 'w': w, 'h': h}
    
    Returns:
        IoU value between 0.0 and 1.0 (1.0 = perfect overlap)
    """
    # Get coordinates
    x1_min = box1.get('x', 0)
    y1_min = box1.get('y', 0)
    x1_max = x1_min + box1.get('w', 0)
    y1_max = y1_min + box1.get('h', 0)
    
    x2_min = box2.get('x', 0)
    y2_min = box2.get('y', 0)
    x2_max = x2_min + box2.get('w', 0)
    y2_max = y2_min + box2.get('h', 0)
    
    # Calculate intersection
    inter_x_min = max(x1_min, x2_min)
    inter_y_min = max(y1_min, y2_min)
    inter_x_max = min(x1_max, x2_max)
    inter_y_max = min(y1_max, y2_max)
    
    if inter_x_max <= inter_x_min or inter_y_max <= inter_y_min:
        return 0.0
    
    inter_area = (inter_x_max - inter_x_min) * (inter_y_max - inter_y_min)
    
    # Calculate union
    box1_area = box1.get('w', 0) * box1.get('h', 0)
    box2_area = box2.get('w', 0) * box2.get('h', 0)
    union_area = box1_area + box2_area - inter_area
    
    if union_area == 0:
        return 0.0
    
    return inter_area / union_area


def _find_matching_pose_info(
    facial_area: Dict, 
    pose_faces: List[Dict], 
    used_pose_indices: Optional[set] = None
) -> Tuple[Dict, Optional[int]]:
    """Match DeepFace result with RetinaFace pose detection result using IoU.
    
    Uses Intersection over Union (IoU) for robust bounding box matching, which is
    the standard approach in computer vision. This is more reliable than center
    point distance, especially when bounding boxes have different sizes or aspect ratios.
    
    Args:
        facial_area: DeepFace facial_area {'x': x, 'y': y, 'w': w, 'h': h}
        pose_faces: List of RetinaFace detection results with pose info
        used_pose_indices: Set of indices of pose_faces that have already been matched
    
    Returns:
        Tuple of (pose_info_dict, matched_pose_face_index):
        - pose_info_dict: Dictionary with pose information, or defaults
        - matched_pose_face_index: Index of matched pose_face in list, or None if no match
    """
    if used_pose_indices is None:
        used_pose_indices = set()
    if not pose_faces:
        return {
            'pose_mode': 'frontal',
            'yaw_angle': None,
            'pitch_angle': None,
            'roll_angle': None,
            'face_width': None
        }, None
    
    # If only one face detected by both systems, use it directly (if not already used)
    if len(pose_faces) == 1 and 0 not in used_pose_indices:
        pose_face = pose_faces[0]
        pose_area = pose_face.get('facial_area', {})
        
        # Handle both dict and list formats
        if isinstance(pose_area, list) and len(pose_area) >= 4:
            pose_area = {
                'x': pose_area[0],
                'y': pose_area[1],
                'w': pose_area[2],
                'h': pose_area[3]
            }
        
        if isinstance(pose_area, dict) and pose_area:
            # Still check IoU to ensure it's a reasonable match
            iou = _calculate_iou(facial_area, pose_area)
            if iou > 0.1:  # At least 10% overlap
                return {
                    'pose_mode': pose_face.get('pose_mode', 'frontal'),
                    'yaw_angle': pose_face.get('yaw_angle'),
                    'pitch_angle': pose_face.get('pitch_angle'),
                    'roll_angle': pose_face.get('roll_angle'),
                    'face_width': pose_face.get('face_width')  # Extract face width
                }, 0  # Return index 0 since it's the only face
    
    # Multiple faces: find best match using IoU (excluding already-used poses)
    best_match = None
    best_match_index = None
    best_iou = 0.0
    
    for idx, pose_face in enumerate(pose_faces):
        # Skip poses that have already been matched to other faces
        if idx in used_pose_indices:
            continue
            
        pose_area = pose_face.get('facial_area', {})
        
        # Handle both dict and list formats
        if isinstance(pose_area, list) and len(pose_area) >= 4:
            pose_area = {
                'x': pose_area[0],
                'y': pose_area[1],
                'w': pose_area[2],
                'h': pose_area[3]
            }
        elif not isinstance(pose_area, dict):
            continue
        
        if not pose_area:
            continue
        
        # Calculate IoU between DeepFace and RetinaFace bounding boxes
        iou = _calculate_iou(facial_area, pose_area)
        
        if iou > best_iou:
            best_iou = iou
            best_match = pose_face
            best_match_index = idx
    
    # Use match if IoU is above threshold (0.1 = 10% overlap is very lenient)
    # Since DeepFace uses RetinaFace as detector_backend, they should detect similar faces
    # Lower threshold to catch more matches
    if best_match and best_iou > 0.1:
        return {
            'pose_mode': best_match.get('pose_mode', 'frontal'),
            'yaw_angle': best_match.get('yaw_angle'),
            'pitch_angle': best_match.get('pitch_angle'),
            'roll_angle': best_match.get('roll_angle'),
            'face_width': best_match.get('face_width')  # Extract face width
        }, best_match_index
    
    # Aggressive fallback: if we have pose_faces detected, use the best match
    # DeepFace and RetinaFace might detect slightly different bounding boxes,
    # but if we have pose data, we should use it
    if best_match:
        deepface_center_x = facial_area.get('x', 0) + facial_area.get('w', 0) / 2
        deepface_center_y = facial_area.get('y', 0) + facial_area.get('h', 0) / 2
        
        pose_area = best_match.get('facial_area', {})
        if isinstance(pose_area, list) and len(pose_area) >= 4:
            pose_area = {
                'x': pose_area[0],
                'y': pose_area[1],
                'w': pose_area[2],
                'h': pose_area[3]
            }
        
        if isinstance(pose_area, dict) and pose_area:
            pose_center_x = pose_area.get('x', 0) + pose_area.get('w', 0) / 2
            pose_center_y = pose_area.get('y', 0) + pose_area.get('h', 0) / 2
            
            distance = ((deepface_center_x - pose_center_x) ** 2 + 
                       (deepface_center_y - pose_center_y) ** 2) ** 0.5
            
            # Very lenient fallback: use if distance is within 30% of face size or 150 pixels
            # This ensures we capture pose data even when bounding boxes differ significantly
            face_size = (facial_area.get('w', 0) + facial_area.get('h', 0)) / 2
            threshold = max(face_size * 0.30, 150.0)  # At least 150 pixels, or 30% of face size
            
            if distance < threshold:
                return {
                    'pose_mode': best_match.get('pose_mode', 'frontal'),
                    'yaw_angle': best_match.get('yaw_angle'),
                    'pitch_angle': best_match.get('pitch_angle'),
                    'roll_angle': best_match.get('roll_angle'),
                    'face_width': best_match.get('face_width')  # Extract face width
                }, best_match_index
            
            # Last resort: if we have pose_faces and only one face, use it regardless
            # This handles cases where DeepFace and RetinaFace detect the same face
            # but with very different bounding boxes
            if len(pose_faces) == 1:
                return {
                    'pose_mode': best_match.get('pose_mode', 'frontal'),
                    'yaw_angle': best_match.get('yaw_angle'),
                    'pitch_angle': best_match.get('pitch_angle'),
                    'roll_angle': best_match.get('roll_angle'),
                    'face_width': best_match.get('face_width')  # Extract face width
                }, best_match_index
    
    # Final fallback: if we have pose_faces but couldn't match, try to extract
    # face_width from the best_match first (if available)
    # This handles cases where RetinaFace detected the face but couldn't calculate
    # yaw (e.g., missing nose landmark), but still has face_width (both eyes present)
    # Or cases where we can't match bounding boxes but have pose data available
    if best_match:
        # Prefer best_match even if IoU was too low - it's still the closest match
        # Calculate center distance to ensure it's at least somewhat close
        deepface_center_x = facial_area.get('x', 0) + facial_area.get('w', 0) / 2
        deepface_center_y = facial_area.get('y', 0) + facial_area.get('h', 0) / 2
        
        pose_area = best_match.get('facial_area', {})
        if isinstance(pose_area, list) and len(pose_area) >= 4:
            pose_area = {
                'x': pose_area[0],
                'y': pose_area[1],
                'w': pose_area[2],
                'h': pose_area[3]
            }
        
        if isinstance(pose_area, dict) and pose_area:
            pose_center_x = pose_area.get('x', 0) + pose_area.get('w', 0) / 2
            pose_center_y = pose_area.get('y', 0) + pose_area.get('h', 0) / 2
            
            distance = ((deepface_center_x - pose_center_x) ** 2 + 
                       (deepface_center_y - pose_center_y) ** 2) ** 0.5
            
            # Only use best_match if distance is reasonable (within 50% of face size or 200 pixels)
            face_size = (facial_area.get('w', 0) + facial_area.get('h', 0)) / 2
            threshold = max(face_size * 0.50, 200.0)  # At least 200 pixels, or 50% of face size
            
            if distance < threshold:
                face_width = best_match.get('face_width')
                if face_width is not None:
                    return {
                        'pose_mode': best_match.get('pose_mode', 'frontal'),
                        'yaw_angle': best_match.get('yaw_angle'),  # May still be None
                        'pitch_angle': best_match.get('pitch_angle'),  # May still be None
                        'roll_angle': best_match.get('roll_angle'),  # May still be None
                        'face_width': face_width  # At least we have this
                    }, best_match_index
    
    # Final fallback: if we have pose_faces but couldn't match, try to extract
    # face_width from any pose_face (even if already used or we can't match it)
    # This allows us to use face_width for profile detection even when matching fails
    # We'll recalculate pose_mode using classify_pose_mode with face_width
    # Note: We check used poses first, then unused ones, to prefer unused poses
    if pose_faces:
        # First try unused poses
        for idx, pose_face in enumerate(pose_faces):
            if idx in used_pose_indices:
                continue
            face_width = pose_face.get('face_width')
            if face_width is not None:
                # Return face_width even if we can't match - pose_mode will be recalculated
                # using classify_pose_mode with face_width
                return {
                    'pose_mode': 'frontal',  # Will be recalculated using face_width
                    'yaw_angle': None,
                    'pitch_angle': None,
                    'roll_angle': None,
                    'face_width': face_width  # At least we have this for classification
                }, None  # Don't mark as used since we didn't match
        
        # If no unused poses with face_width, try used ones as last resort
        # This handles cases where all poses were used but we still need face_width
        for idx, pose_face in enumerate(pose_faces):
            face_width = pose_face.get('face_width')
            if face_width is not None:
                # Return face_width even from used pose - better than nothing
                # pose_mode will be recalculated using classify_pose_mode with face_width
                return {
                    'pose_mode': 'frontal',  # Will be recalculated using face_width
                    'yaw_angle': None,
                    'pitch_angle': None,
                    'roll_angle': None,
                    'face_width': face_width  # At least we have this for classification
                }, None  # Don't mark as used since we didn't match
    
    return {
        'pose_mode': 'frontal',
        'yaw_angle': None,
        'pitch_angle': None,
        'roll_angle': None,
        'face_width': None
    }, None


def process_unprocessed_photos(
    db: Session,
    batch_size: Optional[int] = None,
    detector_backend: str = "retinaface",
    model_name: str = "ArcFace",
    update_progress: Optional[Callable[[int, int, str, int, int], None]] = None,
) -> Tuple[int, int, int]:
    """Process faces in all unprocessed photos.
    
    Args:
        db: Database session
        batch_size: Maximum number of photos to process (None = all)
        detector_backend: DeepFace detector backend
        model_name: DeepFace model name
        update_progress: Optional callback (processed, total, current_file, faces_detected, faces_stored)
        
    Returns:
        Tuple of (photos_processed, total_faces_detected, total_faces_stored)
    """
    # Initialize result variables at function level so they're available in exception handler
    photos_processed = 0
    total_faces_detected = 0
    total_faces_stored = 0
    
    try:
        print(f"[FaceService] Starting face processing: detector={detector_backend}, model={model_name}, batch_size={batch_size}")
        overall_start = time.time()
        
        # Update progress - querying unprocessed photos
        if update_progress:
            try:
                batch_msg = f"Finding up to {batch_size} photos" if batch_size else "Finding photos"
                update_progress(0, 0, f"{batch_msg} that need processing...", 0, 0)
            except Exception as e:
                # If update_progress fails, log but continue
                try:
                    print(f"[FaceService] Warning: update_progress failed: {e}")
                except (BrokenPipeError, OSError):
                    pass
        
        # Desktop parity: find photos that are not yet processed
        # Also filter out videos (only process images for face detection)
        query_start = time.time()
        # Filter for unprocessed photos, excluding videos
        unprocessed_query = db.query(Photo).filter(
            Photo.processed == False,  # noqa: E712
            Photo.media_type != 'video'  # Skip videos (videos are marked as processed and not processed for faces)
        )
        
        # Apply batch size limit BEFORE executing query to avoid loading unnecessary photos
        # When batch_size is set, only that many photos are fetched from the database
        if batch_size:
            unprocessed_query = unprocessed_query.limit(batch_size)
        
        # Execute query - only loads batch_size photos if limit was set
        unprocessed_photos = unprocessed_query.all()
        query_time = time.time() - query_start
        print(f"[FaceService] Query completed in {query_time:.2f}s")
        
        total = len(unprocessed_photos)
        print(f"[FaceService] Found {total} unprocessed photos")
        if total == 0:
            print("[FaceService] No photos to process")
            if update_progress:
                update_progress(0, 0, "No photos to process", 0, 0)
            return 0, 0, 0
        
        # Update progress - preparing to process
        if update_progress:
            update_progress(0, total, f"Preparing to process {total} photos...", 0, 0)
        
        photos_processed = 0
        total_faces_detected = 0
        total_faces_stored = 0
        
        print(f"[FaceService] Starting processing of {total} photos...")
        
        # Helper to check if job was cancelled
        def check_cancelled() -> bool:
            """Check if job has been cancelled by fetching fresh job metadata from Redis.
            
            This function MUST fetch the job fresh from Redis to get the latest cancellation
            status, because the cancellation flag is set by the API endpoint in a
            different process. The cached job object won't see the updated metadata.
            
            Also checks RQ's built-in cancellation status.
            """
            try:
                from rq import get_current_job
                from redis import Redis
                from rq.job import Job
                
                job = get_current_job()
                if not job:
                    return False
                
                job_id = job.id
                
                # Method 1: Check RQ's built-in cancellation status
                # RQ sets this when job.cancel() is called
                try:
                    # Check if job has is_canceled attribute (RQ 1.x)
                    if hasattr(job, 'is_canceled') and job.is_canceled:
                        print(f"[FaceService] ⚠️  STOP: RQ is_canceled=True detected for job {job_id}")
                        return True
                    # Check if job has is_cancelled attribute (some RQ versions)
                    if hasattr(job, 'is_cancelled') and job.is_cancelled:
                        print(f"[FaceService] ⚠️  STOP: RQ is_cancelled=True detected for job {job_id}")
                        return True
                except Exception as cancel_check_error:
                    try:
                        print(f"[FaceService] Debug: Error checking RQ cancellation: {cancel_check_error}")
                    except (BrokenPipeError, OSError):
                        pass
                
                # Method 2: Fetch job fresh from Redis to get latest metadata
                # The cancellation flag is set by the API endpoint in a different process
                try:
                    # Use the job's connection if available, otherwise create new one
                    if hasattr(job, 'connection') and job.connection:
                        connection = job.connection
                    else:
                        connection = Redis(host="localhost", port=6379, db=0, decode_responses=False)
                    
                    # Fetch fresh job from Redis
                    fresh_job = Job.fetch(job_id, connection=connection)
                    
                    # Check metadata flag
                    cancelled = fresh_job.meta and fresh_job.meta.get("cancelled", False)
                    if cancelled:
                        print(f"[FaceService] ⚠️  STOP: CANCELLATION FLAG DETECTED in job {job_id} metadata - stopping after current photo")
                        print(f"[FaceService]   Fresh job meta: {fresh_job.meta}")
                    else:
                        # Debug: log when we check but don't find cancellation (only occasionally to avoid spam)
                        import random
                        if random.random() < 0.01:  # Log 1% of checks
                            print(f"[FaceService] Debug: Checked cancellation for job {job_id}, not cancelled (meta: {fresh_job.meta})")
                    return cancelled
                except Exception as fetch_error:
                    # If fetch fails, try using the cached job object as fallback
                    try:
                        cancelled = job.meta and job.meta.get("cancelled", False)
                        if cancelled:
                            print(f"[FaceService] ⚠️  STOP: CANCELLATION DETECTED (cached metadata) in job {job_id}")
                        return cancelled
                    except Exception:
                        pass
                    # Log the fetch error for debugging
                    try:
                        print(f"[FaceService] Warning: Could not fetch fresh job for cancellation check: {fetch_error}")
                    except (BrokenPipeError, OSError):
                        pass
            except Exception as e:
                # If we can't check, assume not cancelled (fail-safe)
                try:
                    print(f"[FaceService] Warning: Could not check cancellation status: {e}")
                except (BrokenPipeError, OSError):
                    pass
            return False
        
        # Pre-warm DeepFace models BEFORE processing photos
        # This moves the model loading delay to initialization phase (with progress updates)
        # instead of causing delay during first photo processing
        if total > 0:
            print(f"[FaceService] Pre-warming DeepFace models...")
            _pre_warm_deepface(detector_backend, model_name, update_progress)
        
        # Check cancellation after pre-warming
        if check_cancelled():
            print("[FaceService] Job cancelled before processing started")
            return photos_processed, total_faces_detected, total_faces_stored
        
        # Initialize PoseDetector ONCE for the entire batch (reuse across all photos)
        # This avoids reinitializing RetinaFace for every photo, which is very slow
        pose_detector = None
        if RETINAFACE_AVAILABLE:
            try:
                print(f"[FaceService] Initializing RetinaFace pose detector...")
                pose_detector = PoseDetector()
                print(f"[FaceService] Pose detector initialized successfully")
            except Exception as e:
                print(f"[FaceService] ⚠️  Pose detection not available: {e}, will skip pose detection")
                pose_detector = None
        
        # Update progress - models are ready, starting photo processing
        if update_progress and total > 0:
            update_progress(0, total, f"Starting face detection on {total} photos...", 0, 0)
        
        for idx, photo in enumerate(unprocessed_photos, 1):
            # Check for cancellation BEFORE starting each photo
            # This is the primary cancellation point - we stop before starting a new photo
            if check_cancelled():
                print(f"[FaceService] ⚠️  Job cancellation detected at photo {idx}/{total} - stopping before processing")
                if update_progress:
                    try:
                        update_progress(
                            idx - 1,
                            total,
                            "Stopped by user",
                            total_faces_detected,
                            total_faces_stored,
                        )
                    except Exception:
                        pass
                # Raise KeyboardInterrupt to signal cancellation to the task handler
                raise KeyboardInterrupt("Job cancelled by user")
            
            try:
                # Update progress before processing each photo
                if update_progress:
                    update_progress(
                        idx - 1,
                        total,
                        f"Processing {photo.filename}... ({idx}/{total})",
                        total_faces_detected,
                        total_faces_stored,
                    )
                
                # Time the first photo to see if there's still delay after pre-warming
                if idx == 1:
                    first_photo_start = time.time()
                    print(f"[FaceService] Starting first photo processing...")
                
                # Process photo fully (including pose detection, DeepFace, and database commit)
                # CRITICAL: This photo will complete FULLY even if cancellation is requested
                # The cancellation check happens ONLY BEFORE starting this photo and AFTER completing it
                # This ensures no partial data is saved - the entire photo processing (including DB commit)
                # completes before we check for cancellation
                print(f"[FaceService] Processing photo {idx}/{total}: {photo.filename} (cancellation will be checked AFTER this photo completes)")
                faces_detected, faces_stored = process_photo_faces(
                    db,
                    photo,
                    detector_backend=detector_backend,
                    model_name=model_name,
                    pose_detector=pose_detector,  # Reuse the same detector for all photos
                )
                
                # Photo processing is COMPLETE at this point (including DB commit)
                # All faces have been stored in the database
                total_faces_detected += faces_detected
                total_faces_stored += faces_stored
                photos_processed += 1
                
                # Log timing for first photo
                if idx == 1:
                    first_photo_time = time.time() - first_photo_start
                    print(f"[FaceService] First photo completed in {first_photo_time:.2f}s")
                
                # Log completion with explicit confirmation that photo is fully processed
                print(f"[FaceService] ✓ Photo {idx}/{total} ({photo.filename}) FULLY COMPLETED: {faces_detected} faces detected, {faces_stored} faces stored in database")
                
                # Update progress to show completion (including pose detection and DB commit)
                # This happens AFTER the entire photo processing is complete
                if update_progress:
                    try:
                        update_progress(
                            idx,
                            total,
                            f"Completed {photo.filename} ({idx}/{total})",
                            total_faces_detected,
                            total_faces_stored,
                        )
                    except KeyboardInterrupt:
                        # If cancellation was detected during update_progress, check again
                        if check_cancelled():
                            print(f"[FaceService] Job cancelled during progress update after photo {idx}/{total} (photo is already fully processed)")
                            # Raise KeyboardInterrupt to signal cancellation to the task handler
                            raise KeyboardInterrupt("Job cancelled by user after completing current photo")
                        # Re-raise if it wasn't a cancellation
                        raise
                
                # NOW check for cancellation - photo is fully complete (including DB commit)
                # This ensures the entire photo processing is done (pose detection, DeepFace, validation, DB commit),
                # and the progress shows "Completed", then stops before the next one
                if check_cancelled():
                    print(f"[FaceService] ⚠️  STOPPING: Cancellation detected AFTER photo {idx}/{total} ({photo.filename}) fully completed")
                    print(f"[FaceService]   Photo {idx} is safe: {faces_stored} faces stored in database before stopping")
                    # Raise KeyboardInterrupt to signal cancellation to the task handler
                    raise KeyboardInterrupt("Job cancelled by user after completing current photo")
            except KeyboardInterrupt:
                # Cancellation was requested - stop processing gracefully
                print(f"[FaceService] Job cancelled during processing of photo {idx}/{total}")
                if check_cancelled():
                    if update_progress:
                        try:
                            update_progress(
                                idx,
                                total,
                                "Cancelled by user",
                                total_faces_detected,
                                total_faces_stored,
                            )
                        except KeyboardInterrupt:
                            pass
                    break
                # Re-raise if it wasn't a cancellation
                raise
            except Exception as e:
                # Log error but continue processing other photos
                try:
                    print(f"[FaceService] Error processing photo {photo.filename}: {e}")
                except (BrokenPipeError, OSError):
                    pass  # Ignore broken pipe errors
                
                # Try to print traceback, but don't fail if stdout is closed
                try:
                    import traceback
                    traceback.print_exc()
                except (BrokenPipeError, OSError):
                    # If printing fails, at least log the error type
                    try:
                        print(f"[FaceService] Error type: {type(e).__name__}: {str(e)}")
                    except (BrokenPipeError, OSError):
                        pass
                
                # Refresh database session after error to ensure it's in a good state
                # This prevents session state issues from affecting subsequent photos
                # Note: process_photo_faces already does db.rollback(), but we ensure
                # the session is clean for the next photo
                try:
                    # Ensure any partial transaction is rolled back (may be redundant but safe)
                    db.rollback()
                    # Expire the current photo object to clear any stale state
                    db.expire(photo)
                except Exception as session_error:
                    # If session refresh fails, log but don't fail the batch
                    try:
                        print(f"[FaceService] Warning: Could not refresh session after error: {session_error}")
                    except (BrokenPipeError, OSError):
                        pass
                
                if update_progress:
                    try:
                        update_progress(
                            idx,
                            total,
                            f"Error: {photo.filename}",
                            total_faces_detected,
                            total_faces_stored,
                        )
                    except KeyboardInterrupt:
                        # If cancellation detected during error progress update, stop
                        if check_cancelled():
                            print(f"[FaceService] Job cancelled after error on photo {idx}/{total}")
                            break
                
                # Check for cancellation after handling error
                if check_cancelled():
                    print(f"[FaceService] Job cancelled after error on photo {idx}/{total}")
                    if update_progress:
                        try:
                            update_progress(
                                idx,
                                total,
                                "Cancelled by user - finished current photo",
                                total_faces_detected,
                                total_faces_stored,
                            )
                        except KeyboardInterrupt:
                            pass
                    break
    
    except Exception as e:
        # Catch any exceptions that occur outside the photo processing loop
        # (e.g., during initialization, model loading, database queries, etc.)
        try:
            print(f"[FaceService] Fatal error in process_unprocessed_photos: {e}")
            import traceback
            traceback.print_exc()
        except (BrokenPipeError, OSError):
            pass
        
        # Try to update progress with error message
        if update_progress:
            try:
                update_progress(
                    photos_processed,
                    0,
                    f"Fatal error: {str(e)}",
                    total_faces_detected,
                    total_faces_stored,
                )
            except Exception:
                pass
        
        # Re-raise so the task handler can log it properly
        # But we've already logged progress, so the batch can show partial results
        raise
    
    return photos_processed, total_faces_detected, total_faces_stored


def list_unidentified_faces(
    db: Session,
    page: int = 1,
    page_size: int = 50,
    min_quality: float = 0.0,
    date_from: Optional[date] = None,
    date_to: Optional[date] = None,
    date_taken_from: Optional[date] = None,
    date_taken_to: Optional[date] = None,
    date_processed_from: Optional[date] = None,
    date_processed_to: Optional[date] = None,
    sort_by: str = "quality",
    sort_dir: str = "desc",
    tag_names: Optional[List[str]] = None,
    match_all: bool = False,
    photo_ids: Optional[List[int]] = None,
    include_excluded: bool = False,
) -> Tuple[List[Face], int]:
    """Return paginated unidentified faces with filters.

    Supports filtering by:
    - Min quality
    - Date taken (date_taken_from, date_taken_to)
    - Date processed (date_processed_from, date_processed_to) - uses photo.date_added
    - Tags (tag_names, match_all)
    - Photo IDs (photo_ids)
    
    Legacy parameters (date_from, date_to) are kept for backward compatibility
    and filter by date_taken when available, else date_added as fallback.
    """
    # Base query: faces with no person
    query = db.query(Face).join(Photo, Face.photo_id == Photo.id).filter(Face.person_id.is_(None))
    
    # Filter by excluded status (exclude excluded faces by default)
    if not include_excluded:
        query = query.filter(Face.excluded == False)
    
    # Tag filtering
    if tag_names:
        # Find tag IDs (case-insensitive)
        tag_ids = (
            db.query(Tag.id)
            .filter(func.lower(Tag.tag_name).in_([t.lower().strip() for t in tag_names]))
            .all()
        )
        tag_ids = [tid[0] for tid in tag_ids]
        
        if tag_ids:
            if match_all:
                # Photos that have ALL specified tags
                # Join with PhotoTagLinkage and filter
                query = (
                    query.join(PhotoTagLinkage, Photo.id == PhotoTagLinkage.photo_id)
                    .filter(PhotoTagLinkage.tag_id.in_(tag_ids))
                    .group_by(Face.id, Photo.id)
                    .having(func.count(func.distinct(PhotoTagLinkage.tag_id)) == len(tag_ids))
                )
            else:
                # Photos that have ANY of the specified tags
                tagged_photo_ids_subquery = (
                    db.query(PhotoTagLinkage.photo_id)
                    .filter(PhotoTagLinkage.tag_id.in_(tag_ids))
                )
                query = query.filter(Face.photo_id.in_(tagged_photo_ids_subquery))
        else:
            # No matching tags found - return empty result
            return [], 0

    # Photo ID filtering
    if photo_ids:
        query = query.filter(Face.photo_id.in_(photo_ids))

    # Min quality (stored 0.0-1.0)
    if min_quality is not None:
        query = query.filter(Face.quality_score >= min_quality)

    # Date taken filters (new separate filters)
    if date_taken_from is not None:
        query = query.filter(Photo.date_taken >= date_taken_from)
    if date_taken_to is not None:
        query = query.filter(Photo.date_taken <= date_taken_to)
    
    # Date processed filters (uses photo.date_added)
    if date_processed_from is not None:
        query = query.filter(func.date(Photo.date_added) >= date_processed_from)
    if date_processed_to is not None:
        query = query.filter(func.date(Photo.date_added) <= date_processed_to)
    
    # Legacy date filters (backward compatibility)
    # Filter by date_taken when available, else date_added as fallback
    if date_from is not None:
        query = query.filter(
            (Photo.date_taken.is_not(None) & (Photo.date_taken >= date_from))
            | (Photo.date_taken.is_(None) & (func.date(Photo.date_added) >= date_from))
        )
    if date_to is not None:
        query = query.filter(
            (Photo.date_taken.is_not(None) & (Photo.date_taken <= date_to))
            | (Photo.date_taken.is_(None) & (func.date(Photo.date_added) <= date_to))
        )

    # Sorting
    if sort_by == "quality":
        sort_col = Face.quality_score
    elif sort_by == "date_taken":
        sort_col = Photo.date_taken
    else:
        sort_col = Photo.date_added

    if sort_dir == "asc":
        query = query.order_by(sort_col.asc().nullslast())
    else:
        query = query.order_by(sort_col.desc().nullslast())
    
    # Add secondary sort by quality (descending, best first) when primary sort is not quality
    if sort_by != "quality":
        query = query.order_by(Face.quality_score.desc().nullslast())
    
    # Add tertiary sort by pose mode: frontal first, then tilted, then profile last
    # Check profile first to catch poses that might contain both 'tilted' and 'profile'
    pose_order = case(
        (func.lower(Face.pose_mode).like('%profile%'), 2),  # Profile = 2 (last)
        (func.lower(Face.pose_mode).like('%tilted%'), 1),   # Tilted = 1 (second)
        (func.lower(Face.pose_mode).like('frontal%'), 0),    # Frontal = 0 (first)
        else_=2  # Default to last for unknown poses
    )
    query = query.order_by(pose_order.asc())

    # Total count for pagination
    total = query.count()

    # Pagination
    items = query.offset((page - 1) * page_size).limit(page_size).all()
    return items, total


def calculate_cosine_distance(encoding1: np.ndarray, encoding2: np.ndarray) -> float:
    """Calculate cosine distance between two face encodings, matching desktop exactly.
    
    Desktop: _calculate_cosine_similarity returns distance (0 = identical, 2 = opposite)
    This matches the desktop implementation exactly.
    
    Optimized: Avoids unnecessary array creation if inputs are already numpy arrays.
    """
    try:
        # Optimize: Check if already numpy arrays to avoid unnecessary conversions
        # Since we pre-load encodings as 1D arrays from np.frombuffer(), we can use them directly
        if isinstance(encoding1, np.ndarray):
            # Already numpy array - use directly if 1D, otherwise flatten (creates view, not copy)
            enc1 = encoding1 if encoding1.ndim == 1 else encoding1.flatten()
        else:
            # Convert to numpy array only if needed (rare case)
            enc1 = np.asarray(encoding1, dtype=np.float64).flatten()
        
        if isinstance(encoding2, np.ndarray):
            # Already numpy array - use directly if 1D, otherwise flatten (creates view, not copy)
            enc2 = encoding2 if encoding2.ndim == 1 else encoding2.flatten()
        else:
            # Convert to numpy array only if needed (rare case)
            enc2 = np.asarray(encoding2, dtype=np.float64).flatten()
        
        # Check if encodings have the same length (all ArcFace encodings are 512-dim, but check for safety)
        if len(enc1) != len(enc2):
            return 2.0  # Maximum distance on mismatch
        
        # Normalize encodings (matching desktop exactly)
        norm1 = np.linalg.norm(enc1)
        norm2 = np.linalg.norm(enc2)
        
        if norm1 == 0 or norm2 == 0:
            return 2.0
        
        enc1_norm = enc1 / (norm1 + 1e-8)
        enc2_norm = enc2 / (norm2 + 1e-8)
        
        # Calculate cosine similarity
        cosine_sim = np.dot(enc1_norm, enc2_norm)
        
        # Clamp to valid range [-1, 1]
        cosine_sim = np.clip(cosine_sim, -1.0, 1.0)
        
        # Convert to distance (0 = identical, 2 = opposite)
        # For consistency with face_recognition's distance metric
        distance = 1.0 - cosine_sim  # Range [0, 2], where 0 is perfect match
        
        return float(distance)
        
    except Exception as e:
        return 2.0  # Maximum distance on error


def calculate_adaptive_tolerance(base_tolerance: float, face_quality: float) -> float:
    """Calculate adaptive tolerance based on face quality, matching desktop exactly."""
    # Start with base tolerance
    tolerance = base_tolerance
    
    # Adjust based on face quality (higher quality = stricter tolerance)
    quality_factor = 0.9 + (face_quality * 0.2)  # Range: 0.9 to 1.1
    tolerance *= quality_factor
    
    # Ensure tolerance stays within reasonable bounds for DeepFace
    return max(0.2, min(0.6, tolerance))


def calibrate_confidence(distance: float, tolerance: float = None) -> float:
    """Convert distance to calibrated confidence percentage, matching desktop exactly.
    
    Uses empirical calibration method matching desktop _calibrate_confidence.
    Args:
        distance: Cosine distance (0 = identical, 2 = opposite)
        tolerance: Matching tolerance threshold (default: DEFAULT_FACE_TOLERANCE)
    Returns:
        Calibrated confidence percentage (0-100) representing actual match probability
    """
    if tolerance is None:
        tolerance = DEFAULT_FACE_TOLERANCE
    
    # Use configuration setting to determine calibration method (matching desktop)
    if not USE_CALIBRATED_CONFIDENCE:
        # Fallback to simple linear transformation (old behavior)
        return max(0, min(100, (1 - distance) * 100))
    
    if CONFIDENCE_CALIBRATION_METHOD == "linear":
        # Simple linear transformation (old behavior)
        return max(0, min(100, (1 - distance) * 100))
    
    elif CONFIDENCE_CALIBRATION_METHOD == "sigmoid":
        # Sigmoid-based calibration
        normalized_distance = distance / tolerance
        sigmoid_factor = 1 / (1 + np.exp(5 * (normalized_distance - 1)))
        return max(1, min(100, sigmoid_factor * 100))
    
    else:  # "empirical" - default method (matching desktop exactly)
        # Empirical calibration parameters for DeepFace ArcFace model
        # These are derived from analysis of distance distributions for matching/non-matching pairs
        
        # For distances well below threshold: high confidence
        if distance <= tolerance * 0.5:
            # Very close matches: exponential decay from 100%
            confidence = 100 * np.exp(-distance * 2.5)
            return min(100, max(95, confidence))
        
        # For distances near threshold: moderate confidence  
        elif distance <= tolerance:
            # Near-threshold matches: sigmoid-like curve
            # Maps distance to probability based on empirical data
            normalized_distance = (distance - tolerance * 0.5) / (tolerance * 0.5)
            confidence = 95 - (normalized_distance * 40)  # 95% to 55% range
            return max(55, min(95, confidence))
        
        # For distances above threshold: low confidence
        elif distance <= tolerance * 1.5:
            # Above threshold but not too far: rapid decay
            normalized_distance = (distance - tolerance) / (tolerance * 0.5)
            confidence = 55 - (normalized_distance * 35)  # 55% to 20% range
            return max(20, min(55, confidence))
        
        # For very large distances: very low confidence
        else:
            # Very far matches: very low probability
            confidence = 20 * np.exp(-(distance - tolerance * 1.5) * 1.5)
            return max(1, min(20, confidence))


def _is_acceptable_pose_for_auto_match(pose_mode: str) -> bool:
    """Check if pose_mode is acceptable for auto-match (frontal or tilted, but not profile).
    
    Args:
        pose_mode: Pose classification string (e.g., 'frontal', 'tilted_left', 'profile_left')
    
    Returns:
        True if pose is acceptable (frontal or tilted), False if profile or other non-frontal
    """
    if not pose_mode:
        return True  # Default to frontal if None
    
    pose_mode = pose_mode.lower()
    
    # Accept frontal faces
    if pose_mode == 'frontal':
        return True
    
    # Accept tilted faces (but not profile)
    # Check if it contains 'tilted' but NOT 'profile'
    if 'tilted' in pose_mode and 'profile' not in pose_mode:
        return True
    
    # Reject profile faces and other non-frontal poses
    if 'profile' in pose_mode:
        return False
    
    # For other combinations, check if they're still frontal-like
    # (e.g., 'frontal_looking_up' would be acceptable)
    if pose_mode.startswith('frontal'):
        return True
    
    return False


def find_similar_faces(
    db: Session,
    face_id: int,
    limit: int = 20000,  # Very high default limit - effectively unlimited
    tolerance: float = 0.6,  # DEFAULT_FACE_TOLERANCE from desktop
    filter_frontal_only: bool = False,  # New: Only return frontal or tilted faces (not profile)
    include_excluded: bool = False,  # Include excluded faces in results
) -> List[Tuple[Face, float, float]]:  # Returns (face, distance, confidence_pct)
    """Find similar faces matching desktop logic exactly.
    
    Desktop flow:
    1. Get ALL faces from database
    2. Calculate distance for each
    3. Filter by distance <= adaptive_tolerance
    4. Return matches with photo info
    
    Then _get_filtered_similar_faces:
    1. Filters by person_id is None (unidentified)
    2. Filters by confidence >= 40%
    3. Sorts by distance
    
    Args:
        filter_frontal_only: Only return frontal or tilted faces (not profile)
        include_excluded: Include excluded faces in results (default: False)
    """
    from backend.db.models import Photo
    
    if tolerance is None:
        tolerance = DEFAULT_FACE_TOLERANCE
    
    # Get base face - matching desktop
    base: Face = db.query(Face).filter(Face.id == face_id).first()
    if not base:
        return []

    # Load base encoding - desktop uses float64, ArcFace has 512 dimensions
    # Stored as float64: 512 * 8 bytes = 4096 bytes
    base_enc = np.frombuffer(base.encoding, dtype=np.float64)
    base_enc = base_enc.copy()  # Make a copy to avoid buffer issues
    
    # Desktop uses 0.5 as default quality for target face (hardcoded, matching desktop exactly)
    # Desktop: target_quality = 0.5  # Default quality for target face
    base_quality = 0.5

    # Desktop: get ALL faces from database (matching get_all_face_encodings)
    # Desktop find_similar_faces gets ALL faces, doesn't filter by photo_id
    # Get all faces except itself, with photo loaded
    all_faces: List[Face] = (
        db.query(Face)
        .options(joinedload(Face.photo))
        .filter(Face.id != face_id)
        .all()
    )

    matches: List[Tuple[Face, float, float]] = []
    for f in all_faces:
        # Load other encoding - desktop uses float64, ArcFace has 512 dimensions
        other_enc = np.frombuffer(f.encoding, dtype=np.float64)
        other_enc = other_enc.copy()  # Make a copy to avoid buffer issues
        
        other_quality = float(f.quality_score) if f.quality_score is not None else 0.5
        
        # Calculate adaptive tolerance based on both face qualities (matching desktop exactly)
        avg_quality = (base_quality + other_quality) / 2
        adaptive_tolerance = calculate_adaptive_tolerance(tolerance, avg_quality)
        
        # Calculate distance (matching desktop exactly)
        distance = calculate_cosine_distance(base_enc, other_enc)
        
        # Filter by distance <= adaptive_tolerance (matching desktop find_similar_faces)
        if distance <= adaptive_tolerance:
            # Get photo info (desktop does this in find_similar_faces)
            if f.photo:
                # Calculate calibrated confidence (matching desktop _get_filtered_similar_faces)
                confidence_pct = calibrate_confidence(distance, DEFAULT_FACE_TOLERANCE)
                
                # Desktop _get_filtered_similar_faces filters by:
                # 1. person_id is None (unidentified)
                # 2. confidence >= 40%
                is_unidentified = f.person_id is None
                
                if is_unidentified and confidence_pct >= 40:
                    # Filter by excluded status if not including excluded faces
                    if not include_excluded and getattr(f, "excluded", False):
                        continue
                    
                    # Filter by pose_mode if requested (only frontal or tilted faces)
                    if filter_frontal_only and not _is_acceptable_pose_for_auto_match(f.pose_mode):
                        continue
                    
                    # Return calibrated confidence percentage (matching desktop)
                    # Desktop displays confidence_pct directly from _get_calibrated_confidence
                    matches.append((f, distance, confidence_pct))

    # Sort by distance (lower is better) - matching desktop
    matches.sort(key=lambda x: x[1])
    
    # Limit results
    return matches[:limit]


def calculate_batch_similarities(
    db: Session,
    face_ids: list[int],
    min_confidence: float = 60.0,
) -> list[tuple[int, int, float, float]]:
    """Calculate similarities between N faces and all M faces in database.
    
    Optimized implementation using:
    - Phase 1: Pre-normalization of all encodings (avoids repeated normalization)
    - Phase 2: Vectorized batch operations using numpy matrix multiplication
    
    Loads all M faces once from database (cached) and compares each of the N faces
    to all M faces using efficient vectorized operations. Returns list of 
    (face_id_1, face_id_2, similarity, confidence_pct) tuples.
    
    Args:
        db: Database session
        face_ids: List of face IDs to calculate similarities for (N faces)
        min_confidence: Minimum confidence percentage (0-100) to include a pair
    
    Returns:
        List of (face_id_1, face_id_2, similarity, confidence_pct) tuples
        where face_id_1 is from the request list, face_id_2 is from all faces in DB
        similarity is in [0,1] range and confidence_pct is in [0,100] range
    """
    if not face_ids:
        return []
    
    # Load all M faces once from database (cache all faces)
    # Note: Don't load photo data - we only need encodings and quality scores
    all_faces: list[Face] = (
        db.query(Face)
        .all()
    )
    
    # Create a map of all face_id -> Face for quick lookup
    all_face_map = {f.id: f for f in all_faces}
    
    # Load the N faces from the request
    request_faces: list[Face] = [
        all_face_map[fid] for fid in face_ids if fid in all_face_map
    ]
    
    if not request_faces:
        return []
    
    # Pre-load all M encodings and quality scores once (major optimization)
    # This avoids loading from bytes repeatedly in the inner loop
    all_encodings: dict[int, np.ndarray] = {}
    all_qualities: dict[int, float] = {}
    
    for face in all_faces:
        # Pre-load encoding as numpy array
        all_encodings[face.id] = np.frombuffer(face.encoding, dtype=np.float64)
        # Pre-cache quality score
        all_qualities[face.id] = float(face.quality_score) if face.quality_score is not None else 0.5
    
    # Pre-load request face encodings and qualities
    request_encodings: dict[int, np.ndarray] = {}
    request_qualities: dict[int, float] = {}
    
    for face in request_faces:
        request_encodings[face.id] = all_encodings[face.id]
        request_qualities[face.id] = all_qualities[face.id]
    
    # Phase 1: Pre-normalize all encodings (major optimization)
    # This avoids normalizing each encoding thousands of times
    normalized_all_encodings: dict[int, np.ndarray] = {}
    for face_id, enc in all_encodings.items():
        # Normalize encoding once
        norm = np.linalg.norm(enc)
        if norm > 0:
            normalized_all_encodings[face_id] = enc / (norm + 1e-8)
        else:
            # Handle zero-norm edge case
            normalized_all_encodings[face_id] = enc
    
    normalized_request_encodings: dict[int, np.ndarray] = {}
    for face_id, enc in request_encodings.items():
        # Normalize encoding once
        norm = np.linalg.norm(enc)
        if norm > 0:
            normalized_request_encodings[face_id] = enc / (norm + 1e-8)
        else:
            # Handle zero-norm edge case
            normalized_request_encodings[face_id] = enc
    
    # Phase 2: Vectorized batch operations using numpy matrix multiplication
    # Build matrices for vectorized computation
    request_face_ids = list(request_encodings.keys())
    all_face_ids = list(all_encodings.keys())
    
    # Create matrices: (N, 512) and (M, 512) where 512 is encoding dimension
    request_matrix = np.array([normalized_request_encodings[fid] for fid in request_face_ids])
    all_matrix = np.array([normalized_all_encodings[fid] for fid in all_face_ids])
    
    # Calculate all cosine similarities at once using matrix multiplication
    # Shape: (N, 512) @ (512, M) = (N, M)
    # This computes dot product of each request face with each database face
    similarity_matrix = request_matrix @ all_matrix.T
    
    # Clamp to valid range [-1, 1] for cosine similarity
    similarity_matrix = np.clip(similarity_matrix, -1.0, 1.0)
    
    # Convert to distance: distance = 1 - similarity
    # Range: [0, 2] where 0 is perfect match
    distance_matrix = 1.0 - similarity_matrix
    
    # Calculate similarities: filter and process results
    pairs: list[tuple[int, int, float, float]] = []
    tolerance = DEFAULT_FACE_TOLERANCE
    
    # Process results from the distance matrix
    for i, face_id_1 in enumerate(request_face_ids):
        quality_1 = request_qualities[face_id_1]
        
        for j, face_id_2 in enumerate(all_face_ids):
            # Skip comparing face to itself
            if face_id_1 == face_id_2:
                continue
            
            # Get distance from pre-computed matrix
            distance = float(distance_matrix[i, j])
            
            # Get pre-cached quality score
            quality_2 = all_qualities[face_id_2]
            
            # Calculate adaptive tolerance
            avg_quality = (quality_1 + quality_2) / 2
            adaptive_tolerance = calculate_adaptive_tolerance(tolerance, avg_quality)
            
            # Check if within tolerance
            if distance <= adaptive_tolerance:
                # Calculate calibrated confidence
                confidence_pct = calibrate_confidence(distance, tolerance)
                
                # Filter by minimum confidence
                if confidence_pct >= min_confidence:
                    # Convert to similarity (0-1 range, higher = more similar)
                    similarity = 1.0 - (distance / 2.0)  # Normalize distance to [0,1]
                    similarity = max(0.0, min(1.0, similarity))  # Clamp to [0,1]
                    
                    pairs.append((face_id_1, face_id_2, similarity, confidence_pct))
    
    return pairs


def find_auto_match_matches(
    db: Session,
    tolerance: float = 0.6,
    filter_frontal_only: bool = False,
) -> List[Tuple[int, int, Face, List[Tuple[Face, float, float]]]]:
    """Find auto-match matches for all identified people, matching desktop logic exactly.
    
    Desktop flow (from auto_match_panel.py _start_auto_match):
    1. Get all identified faces (one per person, best quality >= 0.3)
    2. Group by person and get best quality face per person
    3. For each person, find similar unidentified faces using _get_filtered_similar_faces
    4. Return matches grouped by person
    
    Args:
        tolerance: Similarity tolerance (default: 0.6)
        filter_frontal_only: Only include persons with frontal or tilted reference face (not profile)
    
    Returns:
        List of (person_id, reference_face_id, reference_face, matches) tuples
        where matches is list of (face, distance, confidence_pct) tuples
    """
    if tolerance is None:
        tolerance = DEFAULT_FACE_TOLERANCE
    
    # Get all identified faces (one per person) to use as reference faces
    # Desktop query:
    # SELECT f.id, f.person_id, f.photo_id, f.location, p.filename, f.quality_score,
    #        f.face_confidence, f.detector_backend, f.model_name
    # FROM faces f
    # JOIN photos p ON f.photo_id = p.id
    # WHERE f.person_id IS NOT NULL AND f.quality_score >= 0.3
    # ORDER BY f.person_id, f.quality_score DESC, pose_mode (frontal first, then tilted, then profile)
    # Add pose mode ordering: frontal first, then tilted, then profile last
    pose_order = case(
        (func.lower(Face.pose_mode).like('%profile%'), 2),  # Profile = 2 (last)
        (func.lower(Face.pose_mode).like('%tilted%'), 1),   # Tilted = 1 (second)
        (func.lower(Face.pose_mode).like('frontal%'), 0),    # Frontal = 0 (first)
        else_=2  # Default to last for unknown poses
    )
    
    identified_faces: List[Face] = (
        db.query(Face)
        .join(Photo, Face.photo_id == Photo.id)
        .filter(Face.person_id.isnot(None))
        .filter(Face.quality_score >= 0.3)
        .order_by(Face.person_id, Face.quality_score.desc(), pose_order.asc())
        .all()
    )
    
    if not identified_faces:
        return []
    
    # Filter by pose_mode if requested (only frontal or tilted faces)
    if filter_frontal_only:
        identified_faces = [
            f for f in identified_faces 
            if _is_acceptable_pose_for_auto_match(f.pose_mode)
        ]
    
    if not identified_faces:
        return []
    
    # Group by person and get the best quality face per person (matching desktop)
    person_faces: Dict[int, Face] = {}
    for face in identified_faces:
        person_id = face.person_id
        if person_id not in person_faces:
            person_faces[person_id] = face
    
    # Convert to ordered list to ensure consistent ordering
    # Desktop sorts by person name for consistent, user-friendly ordering
    person_faces_list = []
    for person_id, face in person_faces.items():
        # Get person name for ordering
        person = db.query(Person).filter(Person.id == person_id).first()
        if person:
            if person.last_name and person.first_name:
                person_name = f"{person.last_name}, {person.first_name}"
            elif person.last_name:
                person_name = person.last_name
            elif person.first_name:
                person_name = person.first_name
            else:
                person_name = "Unknown"
        else:
            person_name = "Unknown"
        person_faces_list.append((person_id, face, person_name))
    
    # Sort by person name for consistent, user-friendly ordering (matching desktop)
    person_faces_list.sort(key=lambda x: x[2])  # Sort by person name (index 2)
    
    # Find similar faces for each identified person (matching desktop)
    results = []
    for person_id, reference_face, person_name in person_faces_list:
        reference_face_id = reference_face.id
        
        # Use find_similar_faces which matches desktop _get_filtered_similar_faces logic
        # Desktop: similar_faces = self.face_processor._get_filtered_similar_faces(
        #     reference_face_id, tolerance, include_same_photo=False, face_status=None)
        # This filters by: person_id is None (unidentified), confidence >= 40%, sorts by distance
        # Auto-match always excludes excluded faces
        similar_faces = find_similar_faces(
            db, reference_face_id, tolerance=tolerance,
            filter_frontal_only=filter_frontal_only,
            include_excluded=False  # Auto-match always excludes excluded faces
        )
        
        if similar_faces:
            results.append((person_id, reference_face_id, reference_face, similar_faces))
    
    return results


def get_auto_match_people_list(
    db: Session,
    filter_frontal_only: bool = False,
    tolerance: float = 0.6,
) -> List[Tuple[int, Face, str, int]]:
    """Get list of people for auto-match (without matches) - fast initial load.
    
    Returns just the people list with reference faces, without calculating matches.
    This allows fast initial page load, then matches can be loaded on-demand.
    
    However, we do a quick check to see if there are any unidentified faces at all.
    If there are no unidentified faces, we return an empty list (no one can have matches).
    
    Args:
        filter_frontal_only: Only include persons with frontal or tilted reference face (not profile)
        tolerance: Similarity tolerance (used to check if there are potential matches)
    
    Returns:
        List of (person_id, reference_face, person_name, face_count) tuples
    """
    if tolerance is None:
        tolerance = DEFAULT_FACE_TOLERANCE
    
    # Quick check: if there are no unidentified faces, no one can have matches
    # This is a fast query that avoids loading people who can't possibly have matches
    unidentified_count = (
        db.query(func.count(Face.id))
        .filter(Face.person_id.is_(None))
        .scalar() or 0
    )
    
    if unidentified_count == 0:
        return []
    
    # Get all identified faces (one per person) to use as reference faces
    # Same logic as find_auto_match_matches but without finding matches
    pose_order = case(
        (func.lower(Face.pose_mode).like('%profile%'), 2),  # Profile = 2 (last)
        (func.lower(Face.pose_mode).like('%tilted%'), 1),   # Tilted = 1 (second)
        (func.lower(Face.pose_mode).like('frontal%'), 0),    # Frontal = 0 (first)
        else_=2  # Default to last for unknown poses
    )
    
    identified_faces: List[Face] = (
        db.query(Face)
        .join(Photo, Face.photo_id == Photo.id)
        .filter(Face.person_id.isnot(None))
        .filter(Face.quality_score >= 0.3)
        .order_by(Face.person_id, Face.quality_score.desc(), pose_order.asc())
        .all()
    )
    
    if not identified_faces:
        return []
    
    # Filter by pose_mode if requested (only frontal or tilted faces)
    if filter_frontal_only:
        identified_faces = [
            f for f in identified_faces 
            if _is_acceptable_pose_for_auto_match(f.pose_mode)
        ]
    
    if not identified_faces:
        return []
    
    # Group by person and get the best quality face per person
    person_faces: Dict[int, Face] = {}
    for face in identified_faces:
        person_id = face.person_id
        if person_id not in person_faces:
            person_faces[person_id] = face
    
    # Convert to ordered list with person names
    person_faces_list = []
    for person_id, face in person_faces.items():
        # Get person name for ordering
        person = db.query(Person).filter(Person.id == person_id).first()
        if person:
            if person.last_name and person.first_name:
                person_name = f"{person.last_name}, {person.first_name}"
            elif person.last_name:
                person_name = person.last_name
            elif person.first_name:
                person_name = person.first_name
            else:
                person_name = "Unknown"
        else:
            person_name = "Unknown"
        
        # Get face count for this person
        face_count = (
            db.query(func.count(Face.id))
            .filter(Face.person_id == person_id)
            .scalar() or 0
        )
        
        person_faces_list.append((person_id, face, person_name, face_count))
    
    # Sort by person name for consistent, user-friendly ordering
    person_faces_list.sort(key=lambda x: x[2])  # Sort by person name (index 2)
    
    return person_faces_list


def get_auto_match_person_matches(
    db: Session,
    person_id: int,
    tolerance: float = 0.6,
    filter_frontal_only: bool = False,
) -> List[Tuple[Face, float, float]]:
    """Get matches for a specific person - for lazy loading.
    
    Args:
        person_id: Person ID to get matches for
        tolerance: Similarity tolerance (default: 0.6)
        filter_frontal_only: Only return frontal or tilted faces (not profile)
    
    Returns:
        List of (face, distance, confidence_pct) tuples
    """
    from backend.db.models import Person, Face
    
    # Get reference face for this person (best quality >= 0.3)
    reference_face = (
        db.query(Face)
        .filter(Face.person_id == person_id)
        .filter(Face.quality_score >= 0.3)
        .order_by(Face.quality_score.desc())
        .first()
    )
    
    if not reference_face:
        return []
    
    # Find similar faces using existing function
    # Auto-match always excludes excluded faces
    similar_faces = find_similar_faces(
        db, reference_face.id, tolerance=tolerance,
        filter_frontal_only=filter_frontal_only,
        include_excluded=False  # Auto-match always excludes excluded faces
    )
    
    return similar_faces


def accept_auto_match_matches(
    db: Session,
    person_id: int,
    face_ids: List[int],
    user_id: int | None = None,
) -> Tuple[int, int]:
    """Accept auto-match matches for a person, matching desktop logic exactly.
    
    Desktop flow (from auto_match_panel.py _save_changes):
    1. For each face_id in face_ids, set person_id on face
    2. Insert person_encodings for each identified face
    3. Update person encodings (remove old, add current)
    
    Args:
        db: Database session
        person_id: Person ID to identify faces with
        face_ids: List of face IDs to identify
        user_id: User ID who is performing the identification (optional)
    
    Returns:
        (identified_count, updated_count) tuple
    """
    from backend.db.models import PersonEncoding
    
    # Validate person exists
    person = db.query(Person).filter(Person.id == person_id).first()
    if not person:
        raise ValueError(f"Person {person_id} not found")
    
    # Get all faces to identify
    faces = db.query(Face).filter(Face.id.in_(face_ids)).all()
    if not faces:
        return (0, 0)
    
    identified_count = 0
    
    # Process each face
    for face in faces:
        # Set person_id on face
        face.person_id = person_id
        if user_id is not None:
            face.identified_by_user_id = user_id
        db.add(face)
        
        # Insert person_encoding (matching desktop)
        pe = PersonEncoding(
            person_id=person_id,
            face_id=face.id,
            encoding=face.encoding,
            quality_score=face.quality_score,
            detector_backend=face.detector_backend,
            model_name=face.model_name,
        )
        db.add(pe)
        identified_count += 1
    
    # Commit changes
    db.commit()
    
    # Update person encodings (matching desktop update_person_encodings)
    # Desktop: removes old encodings, adds current face encodings
    # Delete old encodings
    db.query(PersonEncoding).filter(PersonEncoding.person_id == person_id).delete()
    
    # Add current face encodings (quality_score >= 0.3)
    current_faces = (
        db.query(Face)
        .filter(Face.person_id == person_id)
        .filter(Face.quality_score >= 0.3)
        .all()
    )
    
    for face in current_faces:
        pe = PersonEncoding(
            person_id=person_id,
            face_id=face.id,
            encoding=face.encoding,
            quality_score=face.quality_score,
            detector_backend=face.detector_backend,
            model_name=face.model_name,
        )
        db.add(pe)
    
    db.commit()
    
    return (identified_count, len(current_faces))

