migration to web
This commit is contained in:
parent
94385e3dcc
commit
4c2148f7fc
30
README.md
30
README.md
@ -64,7 +64,23 @@ This creates the SQLite database at `data/punimtag.db` (default). For PostgreSQL
|
||||
|
||||
### Running the Application
|
||||
|
||||
**Terminal 1 - Backend API:**
|
||||
**Prerequisites:**
|
||||
- Redis must be running (for background jobs)
|
||||
```bash
|
||||
# Check if Redis is running
|
||||
redis-cli ping
|
||||
# If not running, start Redis:
|
||||
# On Linux: sudo systemctl start redis
|
||||
# On macOS with Homebrew: brew services start redis
|
||||
# Or run directly: redis-server
|
||||
```
|
||||
|
||||
**Terminal 1 - Redis (if not running as service):**
|
||||
```bash
|
||||
redis-server
|
||||
```
|
||||
|
||||
**Terminal 2 - Backend API:**
|
||||
```bash
|
||||
cd /home/ladmin/Code/punimtag
|
||||
source venv/bin/activate
|
||||
@ -72,7 +88,15 @@ export PYTHONPATH=/home/ladmin/Code/punimtag
|
||||
uvicorn src.web.app:app --host 127.0.0.1 --port 8000
|
||||
```
|
||||
|
||||
**Terminal 2 - Frontend:**
|
||||
**Terminal 3 - RQ Worker (required for photo import):**
|
||||
```bash
|
||||
cd /home/ladmin/Code/punimtag
|
||||
source venv/bin/activate
|
||||
export PYTHONPATH=/home/ladmin/Code/punimtag
|
||||
python -m src.web.worker
|
||||
```
|
||||
|
||||
**Terminal 4 - Frontend:**
|
||||
```bash
|
||||
cd /home/ladmin/Code/punimtag/frontend
|
||||
npm run dev
|
||||
@ -84,6 +108,8 @@ Then open your browser to **http://localhost:3000**
|
||||
- Username: `admin`
|
||||
- Password: `admin`
|
||||
|
||||
**Note:** The RQ worker (Terminal 3) is required for background photo import jobs. Without it, jobs will remain in "Pending" status.
|
||||
|
||||
---
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
28
frontend/src/api/jobs.ts
Normal file
28
frontend/src/api/jobs.ts
Normal file
@ -0,0 +1,28 @@
|
||||
import apiClient from './client'
|
||||
|
||||
export enum JobStatus {
|
||||
PENDING = 'pending',
|
||||
STARTED = 'started',
|
||||
PROGRESS = 'progress',
|
||||
SUCCESS = 'success',
|
||||
FAILURE = 'failure',
|
||||
}
|
||||
|
||||
export interface JobResponse {
|
||||
id: string
|
||||
status: JobStatus
|
||||
progress: number
|
||||
message: string
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
export const jobsApi = {
|
||||
getJob: async (jobId: string): Promise<JobResponse> => {
|
||||
const { data } = await apiClient.get<JobResponse>(
|
||||
`/api/v1/jobs/${jobId}`
|
||||
)
|
||||
return data
|
||||
},
|
||||
}
|
||||
|
||||
76
frontend/src/api/photos.ts
Normal file
76
frontend/src/api/photos.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import apiClient from './client'
|
||||
import { JobResponse } from './jobs'
|
||||
|
||||
export interface PhotoImportRequest {
|
||||
folder_path: string
|
||||
recursive?: boolean
|
||||
}
|
||||
|
||||
export interface PhotoImportResponse {
|
||||
job_id: string
|
||||
message: string
|
||||
folder_path?: string
|
||||
estimated_photos?: number
|
||||
}
|
||||
|
||||
export interface PhotoResponse {
|
||||
id: number
|
||||
path: string
|
||||
filename: string
|
||||
checksum?: string
|
||||
date_added: string
|
||||
date_taken?: string
|
||||
width?: number
|
||||
height?: number
|
||||
mime_type?: string
|
||||
}
|
||||
|
||||
export interface UploadResponse {
|
||||
message: string
|
||||
added: number
|
||||
existing: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
export const photosApi = {
|
||||
importPhotos: async (
|
||||
request: PhotoImportRequest
|
||||
): Promise<PhotoImportResponse> => {
|
||||
const { data } = await apiClient.post<PhotoImportResponse>(
|
||||
'/api/v1/photos/import',
|
||||
request
|
||||
)
|
||||
return data
|
||||
},
|
||||
|
||||
uploadPhotos: async (files: File[]): Promise<UploadResponse> => {
|
||||
const formData = new FormData()
|
||||
files.forEach((file) => {
|
||||
formData.append('files', file)
|
||||
})
|
||||
|
||||
const { data } = await apiClient.post<UploadResponse>(
|
||||
'/api/v1/photos/import/upload',
|
||||
formData,
|
||||
{
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
}
|
||||
)
|
||||
return data
|
||||
},
|
||||
|
||||
getPhoto: async (photoId: number): Promise<PhotoResponse> => {
|
||||
const { data } = await apiClient.get<PhotoResponse>(
|
||||
`/api/v1/photos/${photoId}`
|
||||
)
|
||||
return data
|
||||
},
|
||||
|
||||
streamJobProgress: (jobId: string): EventSource => {
|
||||
const baseURL = import.meta.env.VITE_API_URL || 'http://127.0.0.1:8000'
|
||||
return new EventSource(`${baseURL}/api/v1/jobs/stream/${jobId}`)
|
||||
},
|
||||
}
|
||||
|
||||
@ -1,12 +1,419 @@
|
||||
import { useState, useRef, useCallback, useEffect } from 'react'
|
||||
import { photosApi, PhotoImportRequest } from '../api/photos'
|
||||
import { jobsApi, JobResponse, JobStatus } from '../api/jobs'
|
||||
|
||||
interface JobProgress {
|
||||
id: string
|
||||
status: string
|
||||
progress: number
|
||||
message: string
|
||||
processed?: number
|
||||
total?: number
|
||||
}
|
||||
|
||||
export default function Scan() {
|
||||
const [folderPath, setFolderPath] = useState('')
|
||||
const [recursive, setRecursive] = useState(true)
|
||||
const [isImporting, setIsImporting] = useState(false)
|
||||
const [currentJob, setCurrentJob] = useState<JobResponse | null>(null)
|
||||
const [jobProgress, setJobProgress] = useState<JobProgress | null>(null)
|
||||
const [importResult, setImportResult] = useState<{
|
||||
added?: number
|
||||
existing?: number
|
||||
total?: number
|
||||
} | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const eventSourceRef = useRef<EventSource | null>(null)
|
||||
|
||||
// Cleanup event source on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (eventSourceRef.current) {
|
||||
eventSourceRef.current.close()
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleFolderBrowse = () => {
|
||||
// Note: Browser security prevents direct folder selection
|
||||
// This is a workaround - user must type/paste path
|
||||
// In production, consider using Electron or a file picker library
|
||||
const path = prompt('Enter folder path to scan:')
|
||||
if (path) {
|
||||
setFolderPath(path)
|
||||
}
|
||||
}
|
||||
|
||||
const handleDragOver = useCallback((e: React.DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
}, [])
|
||||
|
||||
const handleDrop = useCallback((e: React.DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
|
||||
const files = Array.from(e.dataTransfer.files)
|
||||
const imageFiles = files.filter((file) =>
|
||||
/\.(jpg|jpeg|png|bmp|tiff|tif)$/i.test(file.name)
|
||||
)
|
||||
|
||||
if (imageFiles.length > 0) {
|
||||
handleUploadFiles(imageFiles)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = e.target.files
|
||||
if (files && files.length > 0) {
|
||||
handleUploadFiles(Array.from(files))
|
||||
}
|
||||
}
|
||||
|
||||
const handleUploadFiles = async (files: File[]) => {
|
||||
setIsImporting(true)
|
||||
setError(null)
|
||||
setImportResult(null)
|
||||
|
||||
try {
|
||||
const result = await photosApi.uploadPhotos(files)
|
||||
setImportResult({
|
||||
added: result.added,
|
||||
existing: result.existing,
|
||||
total: result.added + result.existing,
|
||||
})
|
||||
setIsImporting(false)
|
||||
} catch (err: any) {
|
||||
setError(err.response?.data?.detail || err.message || 'Upload failed')
|
||||
setIsImporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleScanFolder = async () => {
|
||||
if (!folderPath.trim()) {
|
||||
setError('Please enter a folder path')
|
||||
return
|
||||
}
|
||||
|
||||
setIsImporting(true)
|
||||
setError(null)
|
||||
setImportResult(null)
|
||||
setCurrentJob(null)
|
||||
setJobProgress(null)
|
||||
|
||||
try {
|
||||
const request: PhotoImportRequest = {
|
||||
folder_path: folderPath.trim(),
|
||||
recursive,
|
||||
}
|
||||
|
||||
const response = await photosApi.importPhotos(request)
|
||||
setCurrentJob({
|
||||
id: response.job_id,
|
||||
status: JobStatus.PENDING,
|
||||
progress: 0,
|
||||
message: response.message,
|
||||
created_at: new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
})
|
||||
|
||||
// Start SSE stream for job progress
|
||||
startJobProgressStream(response.job_id)
|
||||
} catch (err: any) {
|
||||
setError(err.response?.data?.detail || err.message || 'Import failed')
|
||||
setIsImporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const startJobProgressStream = (jobId: string) => {
|
||||
// Close existing stream if any
|
||||
if (eventSourceRef.current) {
|
||||
eventSourceRef.current.close()
|
||||
}
|
||||
|
||||
const eventSource = photosApi.streamJobProgress(jobId)
|
||||
eventSourceRef.current = eventSource
|
||||
|
||||
eventSource.onmessage = (event) => {
|
||||
try {
|
||||
const data: JobProgress = JSON.parse(event.data)
|
||||
setJobProgress(data)
|
||||
|
||||
// Update job status
|
||||
const statusMap: Record<string, JobStatus> = {
|
||||
pending: JobStatus.PENDING,
|
||||
started: JobStatus.STARTED,
|
||||
progress: JobStatus.PROGRESS,
|
||||
success: JobStatus.SUCCESS,
|
||||
failure: JobStatus.FAILURE,
|
||||
}
|
||||
|
||||
setCurrentJob({
|
||||
id: data.id,
|
||||
status: statusMap[data.status] || JobStatus.PENDING,
|
||||
progress: data.progress,
|
||||
message: data.message,
|
||||
created_at: new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
})
|
||||
|
||||
// Check if job is complete
|
||||
if (data.status === 'success' || data.status === 'failure') {
|
||||
setIsImporting(false)
|
||||
eventSource.close()
|
||||
eventSourceRef.current = null
|
||||
|
||||
// Fetch final job result to get added/existing counts
|
||||
if (data.status === 'success') {
|
||||
fetchJobResult(jobId)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error parsing SSE event:', err)
|
||||
}
|
||||
}
|
||||
|
||||
eventSource.onerror = (err) => {
|
||||
console.error('SSE error:', err)
|
||||
eventSource.close()
|
||||
eventSourceRef.current = null
|
||||
}
|
||||
}
|
||||
|
||||
const fetchJobResult = async (jobId: string) => {
|
||||
try {
|
||||
const job = await jobsApi.getJob(jobId)
|
||||
// Job result may contain added/existing counts in metadata
|
||||
// For now, we'll just update the job status
|
||||
setCurrentJob(job)
|
||||
} catch (err) {
|
||||
console.error('Error fetching job result:', err)
|
||||
}
|
||||
}
|
||||
|
||||
const getStatusColor = (status: JobStatus) => {
|
||||
switch (status) {
|
||||
case JobStatus.SUCCESS:
|
||||
return 'text-green-600'
|
||||
case JobStatus.FAILURE:
|
||||
return 'text-red-600'
|
||||
case JobStatus.STARTED:
|
||||
case JobStatus.PROGRESS:
|
||||
return 'text-blue-600'
|
||||
default:
|
||||
return 'text-gray-600'
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900 mb-4">Scan</h1>
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<p className="text-gray-600">Folder scanning UI coming in Phase 2.</p>
|
||||
<div className="p-6">
|
||||
<h1 className="text-2xl font-bold text-gray-900 mb-6">Scan Photos</h1>
|
||||
|
||||
<div className="space-y-6">
|
||||
{/* Folder Scan Section */}
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">
|
||||
Scan Folder
|
||||
</h2>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<label
|
||||
htmlFor="folder-path"
|
||||
className="block text-sm font-medium text-gray-700 mb-2"
|
||||
>
|
||||
Folder Path
|
||||
</label>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
id="folder-path"
|
||||
type="text"
|
||||
value={folderPath}
|
||||
onChange={(e) => setFolderPath(e.target.value)}
|
||||
placeholder="/path/to/photos"
|
||||
className="flex-1 px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
disabled={isImporting}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleFolderBrowse}
|
||||
disabled={isImporting}
|
||||
className="px-4 py-2 bg-gray-100 text-gray-700 rounded-md hover:bg-gray-200 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
Browse
|
||||
</button>
|
||||
</div>
|
||||
<p className="mt-1 text-sm text-gray-500">
|
||||
Enter the full path to the folder containing photos
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center">
|
||||
<input
|
||||
id="recursive"
|
||||
type="checkbox"
|
||||
checked={recursive}
|
||||
onChange={(e) => setRecursive(e.target.checked)}
|
||||
disabled={isImporting}
|
||||
className="h-4 w-4 text-blue-600 focus:ring-blue-500 border-gray-300 rounded"
|
||||
/>
|
||||
<label
|
||||
htmlFor="recursive"
|
||||
className="ml-2 block text-sm text-gray-700"
|
||||
>
|
||||
Scan subdirectories recursively
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleScanFolder}
|
||||
disabled={isImporting || !folderPath.trim()}
|
||||
className="w-full px-4 py-2 bg-blue-600 text-white rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{isImporting ? 'Scanning...' : 'Start Scan'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* File Upload Section */}
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">
|
||||
Upload Photos
|
||||
</h2>
|
||||
|
||||
<div
|
||||
onDragOver={handleDragOver}
|
||||
onDrop={handleDrop}
|
||||
className="border-2 border-dashed border-gray-300 rounded-lg p-8 text-center hover:border-blue-400 transition-colors"
|
||||
>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
multiple
|
||||
accept="image/*"
|
||||
onChange={handleFileSelect}
|
||||
className="hidden"
|
||||
disabled={isImporting}
|
||||
/>
|
||||
|
||||
<div className="space-y-2">
|
||||
<svg
|
||||
className="mx-auto h-12 w-12 text-gray-400"
|
||||
stroke="currentColor"
|
||||
fill="none"
|
||||
viewBox="0 0 48 48"
|
||||
>
|
||||
<path
|
||||
d="M28 8H12a4 4 0 00-4 4v20m32-12v8m0 0v8a4 4 0 01-4 4H12a4 4 0 01-4-4v-4m32-4l-3.172-3.172a4 4 0 00-5.656 0L28 28M8 32l9.172-9.172a4 4 0 015.656 0L28 28m0 0l4 4m4-24h8m-4-4v8m-12 4h.02"
|
||||
strokeWidth={2}
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
<p className="text-sm text-gray-600">
|
||||
Drag and drop photos here, or{' '}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
disabled={isImporting}
|
||||
className="text-blue-600 hover:text-blue-700 focus:outline-none"
|
||||
>
|
||||
browse
|
||||
</button>
|
||||
</p>
|
||||
<p className="text-xs text-gray-500">
|
||||
Supports: JPG, PNG, BMP, TIFF
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Progress Section */}
|
||||
{(currentJob || jobProgress) && (
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">
|
||||
Import Progress
|
||||
</h2>
|
||||
|
||||
{currentJob && (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<div className="flex justify-between items-center mb-2">
|
||||
<span
|
||||
className={`text-sm font-medium ${getStatusColor(currentJob.status)}`}
|
||||
>
|
||||
{currentJob.status === JobStatus.SUCCESS && '✓ '}
|
||||
{currentJob.status === JobStatus.FAILURE && '✗ '}
|
||||
{currentJob.status.charAt(0).toUpperCase() +
|
||||
currentJob.status.slice(1)}
|
||||
</span>
|
||||
<span className="text-sm text-gray-600">
|
||||
{currentJob.progress}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="w-full bg-gray-200 rounded-full h-2">
|
||||
<div
|
||||
className="bg-blue-600 h-2 rounded-full transition-all duration-300"
|
||||
style={{ width: `${currentJob.progress}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{jobProgress && (
|
||||
<div className="text-sm text-gray-600">
|
||||
{jobProgress.processed !== undefined &&
|
||||
jobProgress.total !== undefined && (
|
||||
<p>
|
||||
Processed: {jobProgress.processed} /{' '}
|
||||
{jobProgress.total}
|
||||
</p>
|
||||
)}
|
||||
{jobProgress.message && (
|
||||
<p className="mt-1">{jobProgress.message}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Results Section */}
|
||||
{importResult && (
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">
|
||||
Import Results
|
||||
</h2>
|
||||
|
||||
<div className="space-y-2 text-sm">
|
||||
{importResult.added !== undefined && (
|
||||
<p className="text-green-600">
|
||||
✓ {importResult.added} new photos added
|
||||
</p>
|
||||
)}
|
||||
{importResult.existing !== undefined && (
|
||||
<p className="text-gray-600">
|
||||
{importResult.existing} photos already in database
|
||||
</p>
|
||||
)}
|
||||
{importResult.total !== undefined && (
|
||||
<p className="text-gray-700 font-medium">
|
||||
Total: {importResult.total} photos
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Error Section */}
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-4">
|
||||
<p className="text-sm text-red-800">{error}</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
|
||||
16
run_worker.sh
Executable file
16
run_worker.sh
Executable file
@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Start RQ worker for PunimTag background jobs
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# Activate virtual environment if it exists
|
||||
if [ -d "venv" ]; then
|
||||
source venv/bin/activate
|
||||
fi
|
||||
|
||||
# Set Python path
|
||||
export PYTHONPATH="$(pwd)"
|
||||
|
||||
# Start worker
|
||||
python -m src.web.worker
|
||||
|
||||
@ -3,12 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
from rq import Queue
|
||||
from rq.job import Job
|
||||
from redis import Redis
|
||||
import json
|
||||
import time
|
||||
|
||||
from src.web.schemas.jobs import JobResponse, JobStatus
|
||||
|
||||
@ -32,7 +34,7 @@ def get_job(job_id: str) -> JobResponse:
|
||||
}
|
||||
job_status = status_map.get(job.get_status(), JobStatus.PENDING)
|
||||
progress = 0
|
||||
if job_status == JobStatus.STARTED:
|
||||
if job_status == JobStatus.STARTED or job_status == JobStatus.PROGRESS:
|
||||
progress = job.meta.get("progress", 0) if job.meta else 0
|
||||
elif job_status == JobStatus.SUCCESS:
|
||||
progress = 100
|
||||
@ -43,7 +45,9 @@ def get_job(job_id: str) -> JobResponse:
|
||||
progress=progress,
|
||||
message=message,
|
||||
created_at=datetime.fromisoformat(str(job.created_at)),
|
||||
updated_at=datetime.fromisoformat(str(job.ended_at or job.started_at or job.created_at)),
|
||||
updated_at=datetime.fromisoformat(
|
||||
str(job.ended_at or job.started_at or job.created_at)
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
@ -51,3 +55,64 @@ def get_job(job_id: str) -> JobResponse:
|
||||
detail=f"Job {job_id} not found",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/stream/{job_id}")
|
||||
def stream_job_progress(job_id: str):
|
||||
"""Stream job progress via Server-Sent Events (SSE)."""
|
||||
|
||||
def event_generator():
|
||||
"""Generate SSE events for job progress."""
|
||||
last_progress = -1
|
||||
last_message = ""
|
||||
|
||||
while True:
|
||||
try:
|
||||
job = Job.fetch(job_id, connection=redis_conn)
|
||||
status_map = {
|
||||
"queued": JobStatus.PENDING,
|
||||
"started": JobStatus.STARTED,
|
||||
"finished": JobStatus.SUCCESS,
|
||||
"failed": JobStatus.FAILURE,
|
||||
}
|
||||
job_status = status_map.get(job.get_status(), JobStatus.PENDING)
|
||||
|
||||
progress = 0
|
||||
if job_status == JobStatus.STARTED or job_status == JobStatus.PROGRESS:
|
||||
progress = job.meta.get("progress", 0) if job.meta else 0
|
||||
elif job_status == JobStatus.SUCCESS:
|
||||
progress = 100
|
||||
elif job_status == JobStatus.FAILURE:
|
||||
progress = 0
|
||||
|
||||
message = job.meta.get("message", "") if job.meta else ""
|
||||
|
||||
# Only send event if progress or message changed
|
||||
if progress != last_progress or message != last_message:
|
||||
event_data = {
|
||||
"id": job.id,
|
||||
"status": job_status.value,
|
||||
"progress": progress,
|
||||
"message": message,
|
||||
"processed": job.meta.get("processed", 0) if job.meta else 0,
|
||||
"total": job.meta.get("total", 0) if job.meta else 0,
|
||||
}
|
||||
|
||||
yield f"data: {json.dumps(event_data)}\n\n"
|
||||
last_progress = progress
|
||||
last_message = message
|
||||
|
||||
# Stop streaming if job is complete or failed
|
||||
if job_status in (JobStatus.SUCCESS, JobStatus.FAILURE):
|
||||
break
|
||||
|
||||
time.sleep(0.5) # Poll every 500ms
|
||||
|
||||
except Exception as e:
|
||||
error_data = {"error": str(e)}
|
||||
yield f"data: {json.dumps(error_data)}\n\n"
|
||||
break
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(), media_type="text/event-stream"
|
||||
)
|
||||
|
||||
|
||||
@ -2,7 +2,27 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from rq import Queue
|
||||
from redis import Redis
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.web.db.session import get_db
|
||||
|
||||
# Redis connection for RQ
|
||||
redis_conn = Redis(host="localhost", port=6379, db=0, decode_responses=False)
|
||||
queue = Queue(connection=redis_conn)
|
||||
from src.web.schemas.photos import (
|
||||
PhotoImportRequest,
|
||||
PhotoImportResponse,
|
||||
PhotoResponse,
|
||||
)
|
||||
from src.web.services.photo_service import (
|
||||
find_photos_in_folder,
|
||||
import_photo_from_path,
|
||||
)
|
||||
from src.web.services.tasks import import_photos_task
|
||||
|
||||
router = APIRouter(prefix="/photos", tags=["photos"])
|
||||
|
||||
@ -13,17 +33,118 @@ def list_photos() -> dict:
|
||||
return {"message": "Photos endpoint - to be implemented in Phase 2"}
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
def import_photos() -> dict:
|
||||
"""Import photos - placeholder for Phase 2."""
|
||||
return {"message": "Import endpoint - to be implemented in Phase 2"}
|
||||
@router.post("/import", response_model=PhotoImportResponse)
|
||||
def import_photos(
|
||||
request: PhotoImportRequest,
|
||||
) -> PhotoImportResponse:
|
||||
"""Import photos from a folder path.
|
||||
|
||||
This endpoint enqueues a background job to scan and import photos.
|
||||
"""
|
||||
if not request.folder_path:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="folder_path is required",
|
||||
)
|
||||
|
||||
# Validate folder exists
|
||||
import os
|
||||
|
||||
if not os.path.isdir(request.folder_path):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Folder not found: {request.folder_path}",
|
||||
)
|
||||
|
||||
# Estimate number of photos (quick scan)
|
||||
estimated_photos = len(find_photos_in_folder(request.folder_path, request.recursive))
|
||||
|
||||
# Enqueue job
|
||||
job = queue.enqueue(
|
||||
import_photos_task,
|
||||
request.folder_path,
|
||||
request.recursive,
|
||||
job_timeout="1h", # Allow up to 1 hour for large imports
|
||||
)
|
||||
|
||||
return PhotoImportResponse(
|
||||
job_id=job.id,
|
||||
message=f"Photo import job queued for {request.folder_path}",
|
||||
folder_path=request.folder_path,
|
||||
estimated_photos=estimated_photos,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{photo_id}")
|
||||
def get_photo(photo_id: int) -> dict:
|
||||
"""Get photo by ID - placeholder for Phase 2."""
|
||||
@router.post("/import/upload")
|
||||
async def upload_photos(
|
||||
files: list[UploadFile] = File(...),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Upload photo files directly.
|
||||
|
||||
This endpoint accepts file uploads and imports them immediately.
|
||||
Files are saved to PHOTO_STORAGE_DIR before import.
|
||||
For large batches, prefer the /import endpoint with folder_path.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from src.web.settings import PHOTO_STORAGE_DIR
|
||||
|
||||
# Ensure storage directory exists
|
||||
storage_dir = Path(PHOTO_STORAGE_DIR)
|
||||
storage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
added_count = 0
|
||||
existing_count = 0
|
||||
errors = []
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
# Generate unique filename to avoid conflicts
|
||||
import uuid
|
||||
|
||||
file_ext = Path(file.filename).suffix
|
||||
unique_filename = f"{uuid.uuid4()}{file_ext}"
|
||||
stored_path = storage_dir / unique_filename
|
||||
|
||||
# Save uploaded file to storage
|
||||
content = await file.read()
|
||||
with open(stored_path, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
# Import photo from stored location
|
||||
photo, is_new = import_photo_from_path(db, str(stored_path))
|
||||
if is_new:
|
||||
added_count += 1
|
||||
else:
|
||||
existing_count += 1
|
||||
# If photo already exists, delete duplicate upload
|
||||
if os.path.exists(stored_path):
|
||||
os.remove(stored_path)
|
||||
except Exception as e:
|
||||
errors.append(f"Error uploading {file.filename}: {str(e)}")
|
||||
|
||||
return {
|
||||
"message": f"Get photo {photo_id} - to be implemented in Phase 2",
|
||||
"id": photo_id,
|
||||
"message": f"Uploaded {len(files)} files",
|
||||
"added": added_count,
|
||||
"existing": existing_count,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{photo_id}", response_model=PhotoResponse)
|
||||
def get_photo(photo_id: int, db: Session = Depends(get_db)) -> PhotoResponse:
|
||||
"""Get photo by ID."""
|
||||
from src.web.db.models import Photo
|
||||
|
||||
photo = db.query(Photo).filter(Photo.id == photo_id).first()
|
||||
if not photo:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Photo {photo_id} not found",
|
||||
)
|
||||
|
||||
return PhotoResponse.model_validate(photo)
|
||||
|
||||
|
||||
46
src/web/schemas/photos.py
Normal file
46
src/web/schemas/photos.py
Normal file
@ -0,0 +1,46 @@
|
||||
"""Photo schemas."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class PhotoImportRequest(BaseModel):
|
||||
"""Request to import photos from a folder or upload files."""
|
||||
|
||||
folder_path: Optional[str] = Field(
|
||||
None, description="Path to folder to scan for photos"
|
||||
)
|
||||
recursive: bool = Field(
|
||||
True, description="Whether to scan subdirectories recursively"
|
||||
)
|
||||
|
||||
|
||||
class PhotoResponse(BaseModel):
|
||||
"""Photo response schema."""
|
||||
|
||||
id: int
|
||||
path: str
|
||||
filename: str
|
||||
checksum: Optional[str] = None
|
||||
date_added: datetime
|
||||
date_taken: Optional[datetime] = None
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
mime_type: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class PhotoImportResponse(BaseModel):
|
||||
"""Response after initiating photo import."""
|
||||
|
||||
job_id: str
|
||||
message: str
|
||||
folder_path: Optional[str] = None
|
||||
estimated_photos: Optional[int] = None
|
||||
|
||||
193
src/web/services/photo_service.py
Normal file
193
src/web/services/photo_service.py
Normal file
@ -0,0 +1,193 @@
|
||||
"""Photo import and management services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.core.config import SUPPORTED_IMAGE_FORMATS
|
||||
from src.web.db.models import Photo
|
||||
|
||||
|
||||
def compute_checksum(file_path: str) -> str:
|
||||
"""Compute SHA256 checksum of a file."""
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(file_path, "rb") as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
def extract_exif_date(image_path: str) -> Optional[datetime]:
|
||||
"""Extract date taken from photo EXIF data."""
|
||||
try:
|
||||
with Image.open(image_path) as image:
|
||||
exifdata = image.getexif()
|
||||
|
||||
# Look for date taken in EXIF tags
|
||||
date_tags = [
|
||||
306, # DateTime
|
||||
36867, # DateTimeOriginal
|
||||
36868, # DateTimeDigitized
|
||||
]
|
||||
|
||||
for tag_id in date_tags:
|
||||
if tag_id in exifdata:
|
||||
date_str = exifdata[tag_id]
|
||||
if date_str:
|
||||
# Parse EXIF date format (YYYY:MM:DD HH:MM:SS)
|
||||
try:
|
||||
return datetime.strptime(date_str, "%Y:%m:%d %H:%M:%S")
|
||||
except ValueError:
|
||||
# Try alternative format
|
||||
try:
|
||||
return datetime.strptime(
|
||||
date_str, "%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_image_metadata(image_path: str) -> Tuple[Optional[int], Optional[int], Optional[str]]:
|
||||
"""Get image dimensions and MIME type."""
|
||||
try:
|
||||
with Image.open(image_path) as image:
|
||||
width, height = image.size
|
||||
mime_type = mimetypes.guess_type(image_path)[0] or f"image/{image.format.lower() if image.format else 'unknown'}"
|
||||
return width, height, mime_type
|
||||
except Exception:
|
||||
return None, None, None
|
||||
|
||||
|
||||
def find_photos_in_folder(folder_path: str, recursive: bool = True) -> list[str]:
|
||||
"""Find all photo files in a folder."""
|
||||
folder_path = os.path.abspath(folder_path)
|
||||
if not os.path.isdir(folder_path):
|
||||
return []
|
||||
|
||||
found_photos = []
|
||||
|
||||
if recursive:
|
||||
for root, _dirs, files in os.walk(folder_path):
|
||||
for file in files:
|
||||
file_ext = Path(file).suffix.lower()
|
||||
if file_ext in SUPPORTED_IMAGE_FORMATS:
|
||||
photo_path = os.path.join(root, file)
|
||||
found_photos.append(photo_path)
|
||||
else:
|
||||
for file in os.listdir(folder_path):
|
||||
file_ext = Path(file).suffix.lower()
|
||||
if file_ext in SUPPORTED_IMAGE_FORMATS:
|
||||
photo_path = os.path.join(folder_path, file)
|
||||
if os.path.isfile(photo_path):
|
||||
found_photos.append(photo_path)
|
||||
|
||||
return found_photos
|
||||
|
||||
|
||||
def import_photo_from_path(
|
||||
db: Session, photo_path: str, update_progress: Optional[Callable[[int, int, str], None]] = None
|
||||
) -> Tuple[Optional[Photo], bool]:
|
||||
"""Import a single photo from file path into database.
|
||||
|
||||
Returns:
|
||||
Tuple of (Photo instance or None, is_new: bool)
|
||||
"""
|
||||
photo_path = os.path.abspath(photo_path)
|
||||
filename = os.path.basename(photo_path)
|
||||
|
||||
# Check if photo already exists by path
|
||||
existing = db.query(Photo).filter(Photo.path == photo_path).first()
|
||||
if existing:
|
||||
return existing, False
|
||||
|
||||
# Compute checksum
|
||||
try:
|
||||
checksum = compute_checksum(photo_path)
|
||||
# Check if photo with same checksum exists
|
||||
existing_by_checksum = (
|
||||
db.query(Photo).filter(Photo.checksum == checksum).first()
|
||||
if checksum
|
||||
else None
|
||||
)
|
||||
if existing_by_checksum:
|
||||
return existing_by_checksum, False
|
||||
except Exception:
|
||||
checksum = None
|
||||
|
||||
# Extract metadata
|
||||
date_taken = extract_exif_date(photo_path)
|
||||
width, height, mime_type = get_image_metadata(photo_path)
|
||||
|
||||
# Create new photo record
|
||||
photo = Photo(
|
||||
path=photo_path,
|
||||
filename=filename,
|
||||
checksum=checksum,
|
||||
date_taken=date_taken,
|
||||
width=width,
|
||||
height=height,
|
||||
mime_type=mime_type,
|
||||
)
|
||||
|
||||
db.add(photo)
|
||||
db.commit()
|
||||
db.refresh(photo)
|
||||
|
||||
return photo, True
|
||||
|
||||
|
||||
def import_photos_from_folder(
|
||||
db: Session,
|
||||
folder_path: str,
|
||||
recursive: bool = True,
|
||||
update_progress: Optional[Callable[[int, int, str], None]] = None,
|
||||
) -> Tuple[int, int]:
|
||||
"""Import all photos from a folder.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_path: Path to folder to scan
|
||||
recursive: Whether to scan subdirectories
|
||||
update_progress: Optional callback(processed, total, current_file)
|
||||
|
||||
Returns:
|
||||
Tuple of (added_count, existing_count)
|
||||
"""
|
||||
found_photos = find_photos_in_folder(folder_path, recursive)
|
||||
total = len(found_photos)
|
||||
|
||||
if total == 0:
|
||||
return 0, 0
|
||||
|
||||
added_count = 0
|
||||
existing_count = 0
|
||||
|
||||
for idx, photo_path in enumerate(found_photos, 1):
|
||||
try:
|
||||
photo, is_new = import_photo_from_path(db, photo_path)
|
||||
if is_new:
|
||||
added_count += 1
|
||||
else:
|
||||
existing_count += 1
|
||||
|
||||
if update_progress:
|
||||
update_progress(idx, total, os.path.basename(photo_path))
|
||||
except Exception:
|
||||
# Log error but continue
|
||||
if update_progress:
|
||||
update_progress(idx, total, f"Error: {os.path.basename(photo_path)}")
|
||||
|
||||
return added_count, existing_count
|
||||
|
||||
74
src/web/services/tasks.py
Normal file
74
src/web/services/tasks.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""RQ worker tasks for PunimTag."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from rq import get_current_job
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.web.db.session import SessionLocal
|
||||
from src.web.services.photo_service import import_photos_from_folder
|
||||
|
||||
|
||||
def import_photos_task(folder_path: str, recursive: bool = True) -> dict:
|
||||
"""RQ task to import photos from a folder.
|
||||
|
||||
Updates job metadata with progress:
|
||||
- progress: 0-100
|
||||
- message: status message
|
||||
- processed: number of photos processed
|
||||
- total: total photos found
|
||||
- added: number of new photos added
|
||||
- existing: number of photos that already existed
|
||||
"""
|
||||
job = get_current_job()
|
||||
if not job:
|
||||
raise RuntimeError("Not running in RQ job context")
|
||||
|
||||
db: Session = SessionLocal()
|
||||
|
||||
try:
|
||||
def update_progress(processed: int, total: int, current_file: str) -> None:
|
||||
"""Update job progress."""
|
||||
if job:
|
||||
progress = int((processed / total) * 100) if total > 0 else 0
|
||||
job.meta = {
|
||||
"progress": progress,
|
||||
"message": f"Processing {current_file}... ({processed}/{total})",
|
||||
"processed": processed,
|
||||
"total": total,
|
||||
}
|
||||
job.save_meta()
|
||||
|
||||
# Import photos
|
||||
added, existing = import_photos_from_folder(
|
||||
db, folder_path, recursive, update_progress
|
||||
)
|
||||
|
||||
# Final update
|
||||
total_processed = added + existing
|
||||
result = {
|
||||
"folder_path": folder_path,
|
||||
"recursive": recursive,
|
||||
"added": added,
|
||||
"existing": existing,
|
||||
"total": total_processed,
|
||||
}
|
||||
|
||||
if job:
|
||||
job.meta = {
|
||||
"progress": 100,
|
||||
"message": f"Completed: {added} new, {existing} existing",
|
||||
"processed": total_processed,
|
||||
"total": total_processed,
|
||||
"added": added,
|
||||
"existing": existing,
|
||||
}
|
||||
job.save_meta()
|
||||
|
||||
return result
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
"""Application settings for PunimTag Web."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
APP_TITLE = "PunimTag Web API"
|
||||
APP_VERSION = "0.1.0"
|
||||
|
||||
# Photo storage settings
|
||||
PHOTO_STORAGE_DIR = os.getenv("PHOTO_STORAGE_DIR", "data/uploads")
|
||||
|
||||
|
||||
|
||||
@ -1,20 +1,39 @@
|
||||
"""RQ worker entrypoint for PunimTag."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import signal
|
||||
import sys
|
||||
from typing import NoReturn
|
||||
|
||||
from rq import Worker
|
||||
from rq.connections import use_connection
|
||||
from redis import Redis
|
||||
|
||||
from src.web.services.tasks import import_photos_task
|
||||
|
||||
# Redis connection for RQ
|
||||
redis_conn = Redis(host="localhost", port=6379, db=0, decode_responses=False)
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
"""Worker entrypoint placeholder (RQ/Celery to be wired)."""
|
||||
"""Worker entrypoint - starts RQ worker to process background jobs."""
|
||||
def _handle_sigterm(_signum, _frame):
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
signal.signal(signal.SIGINT, _handle_sigterm)
|
||||
|
||||
# Placeholder: actual worker loop will be implemented in Phase 2.
|
||||
signal.pause()
|
||||
# Register tasks with worker
|
||||
# Tasks are imported from services.tasks
|
||||
worker = Worker(
|
||||
["default"],
|
||||
connection=redis_conn,
|
||||
name="punimtag-worker",
|
||||
)
|
||||
|
||||
# Start worker
|
||||
worker.work()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user