diff --git a/.opencode/skills/code-quality/SKILL.md b/.opencode/skills/code-quality/SKILL.md new file mode 100644 index 0000000..30e58ee --- /dev/null +++ b/.opencode/skills/code-quality/SKILL.md @@ -0,0 +1,145 @@ +--- +name: orchestrator-code-quality +description: Generate a code quality report for the orchestrator/ folder focusing on duplication, complexity hotspots, and error/logging consistency, using npm run check:all as the primary gate; report only (no code changes). +license: MIT +compatibility: opencode +metadata: + audience: maintainers + repo_area: orchestrator + primary_command: npm run check:all +--- + +## What I do + +- Operate strictly within the `orchestrator/` folder. +- Produce a Markdown report describing: + 1. **Duplication / near-duplication** + 2. **Over-complex functions / “god modules”** + 3. **Inconsistent error handling and logging** +- Use `npm run check:all` as the authoritative signal for tests + linting. +- Do **not** change code, configuration, scripts, or dependencies. This skill outputs a report only. + +## When to use me + +Use this when you want an evidence-driven code quality assessment and a prioritized plan for remediation **without implementing changes yet**. + +## Constraints + +- Work only in `orchestrator/`. Do not analyze or reference other folders except when required to understand imports into `orchestrator/`. +- Do not propose changes to tooling. Assume `npm run check:all` is the existing gate. +- Do not perform refactors. Do not open PRs. Do not rewrite files. Report findings and recommended fixes only. + +## Inputs I expect + +- Repository access with an `orchestrator/` directory. +- Ability to run `npm run check:all` from within `orchestrator/`. +- (Optional but recommended) Output logs from `npm run check:all` if the environment cannot run commands. + +## Process + +1. **Set working area** + - Ensure all investigation is scoped to `orchestrator/`. + +2. **Execute quality gate** + - Run `npm run check:all` in `orchestrator/`. + - Capture: + - lint errors/warnings (rule IDs, counts, file paths) + - failing tests (test names, stack traces, determinism vs flakiness indicators) + - coverage output (if produced by the command) + +3. **Assess duplication** + - Identify copy/paste and near-duplicate logic patterns in `orchestrator/`: + - repeated parsing/validation + - repeated error mapping and response shaping + - repeated API call wrappers / fetch patterns + - repeated UI hooks/components with minor differences (if applicable) + - Prefer evidence: + - “same structure appears in files A/B/C” + - “multiple implementations differ only by X/Y flags” + - If the repo already includes duplication tooling, use it; otherwise rely on code inspection and minimal search. + +4. **Assess complexity hotspots** + - Identify functions/modules with: + - deep nesting + - long functions + - mixed responsibilities (validation + IO + transformation + logging) + - difficult-to-test implicit dependencies + - Provide concrete hotspots (file paths, function names) and why they’re hotspots. + +5. **Assess error handling and logging consistency** + - Check for: + - multiple error shapes (different status codes/messages for same category) + - swallowed errors vs raw throws + - missing context/correlation fields in logs + - potentially sensitive data in logs + - Identify boundary points (handlers, adapters) where standardization should occur. + +6. **Synthesize a fix plan** + - Provide a prioritized plan with blast radius assessment and test needs. + - Do not implement. Do not request changes. Report recommended actions only. + +## Report output format + +Create a single Markdown report containing these sections: + +### 1. Scope and constraints + +- Confirm `orchestrator/` scope and reliance on `npm run check:all`. + +### 2. Evidence summary + +- `npm run check:all` outcome: + - Lint: top rule IDs + counts + representative file paths + - Tests: failing tests list + deterministic vs flaky notes + - Coverage: summary if present + +### 3. What’s wrong and why it matters + +Include these subsections (at minimum): + +#### 3.1 Duplication and near-duplication + +- Symptoms observed (with file-level examples) +- Why it matters in this repo +- What should be done to fix it (safe refactor tactics + tests needed) + +#### 3.2 Over-complex functions and “god modules” + +- Hotspots (file paths + functions) +- Risk explanation (bug surface, change friction) +- What should be done (decomposition targets + boundaries + tests) + +#### 3.3 Inconsistent error handling and logging + +- Inconsistencies found (examples) +- Operational/security impact +- What should be done (standard error types/mapping + logging conventions) + +### 4. Fix strategy (prioritized) + +- Phase 1 — Reduce duplication safely +- Phase 2 — Decrease complexity in hotspots +- Phase 3 — Standardize error handling and logging + +Each phase must include: + +- Ordered list of actions +- Expected blast radius (low/medium/high) +- Required test additions (if any) +- Validation steps (how to confirm improvement via `npm run check:all`) + +### 5. Definition of done + +- `npm run check:all` passes reliably +- Measurable reduction in targeted duplication clusters +- Complexity hotspots reduced or isolated behind clear boundaries +- Error/logging behavior consistent at boundaries +- No coverage regressions (if coverage is tracked) + +## Quality bar for recommendations + +- Be specific: include file paths, function names, rule IDs, and observed patterns. +- Be conservative: prefer small, behavior-preserving refactors in the plan. +- Avoid “make it green” shortcuts: + - do not recommend disabling lint rules/tests as a first move + - if quarantining a flaky test is suggested, include a concrete follow-up requirement and deadline policy diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index 9368e91..a300cdd 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -8,6 +8,7 @@ import type { ApplicationStage, ApplicationTask, AppSettings, + BackupInfo, CreateJobInput, Job, JobOutcome, @@ -474,3 +475,25 @@ export async function updateVisaSponsorList(): Promise<{ } // Bulk operations (intentionally none - processing is manual) + +// Backup API +export interface BackupListResponse { + backups: BackupInfo[]; + nextScheduled: string | null; +} + +export async function getBackups(): Promise { + return fetchApi("/backups"); +} + +export async function createManualBackup(): Promise { + return fetchApi("/backups", { + method: "POST", + }); +} + +export async function deleteBackup(filename: string): Promise { + await fetchApi(`/backups/${encodeURIComponent(filename)}`, { + method: "DELETE", + }); +} diff --git a/orchestrator/src/client/pages/SettingsPage.test.tsx b/orchestrator/src/client/pages/SettingsPage.test.tsx index 9e4de40..dcfb95c 100644 --- a/orchestrator/src/client/pages/SettingsPage.test.tsx +++ b/orchestrator/src/client/pages/SettingsPage.test.tsx @@ -114,6 +114,16 @@ const baseSettings: AppSettings = { webhookSecretHint: null, basicAuthActive: false, rxresumeBaseResumeId: null, + // Backup settings + backupEnabled: false, + defaultBackupEnabled: false, + overrideBackupEnabled: null, + backupHour: 2, + defaultBackupHour: 2, + overrideBackupHour: null, + backupMaxCount: 5, + defaultBackupMaxCount: 5, + overrideBackupMaxCount: null, }; const renderPage = () => { diff --git a/orchestrator/src/client/pages/SettingsPage.tsx b/orchestrator/src/client/pages/SettingsPage.tsx index cd7a2fa..3728c7e 100644 --- a/orchestrator/src/client/pages/SettingsPage.tsx +++ b/orchestrator/src/client/pages/SettingsPage.tsx @@ -1,5 +1,6 @@ import * as api from "@client/api"; import { PageHeader } from "@client/components/layout"; +import { BackupSettingsSection } from "@client/pages/settings/components/BackupSettingsSection"; import { DangerZoneSection } from "@client/pages/settings/components/DangerZoneSection"; import { DisplaySettingsSection } from "@client/pages/settings/components/DisplaySettingsSection"; import { EnvironmentSettingsSection } from "@client/pages/settings/components/EnvironmentSettingsSection"; @@ -22,13 +23,14 @@ import { } from "@shared/settings-schema"; import type { AppSettings, + BackupInfo, JobStatus, ResumeProjectCatalogItem, ResumeProjectsSettings, } from "@shared/types"; import { Settings } from "lucide-react"; import type React from "react"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; import { FormProvider, type Resolver, useForm } from "react-hook-form"; import { toast } from "sonner"; import { Accordion } from "@/components/ui/accordion"; @@ -67,6 +69,9 @@ const DEFAULT_FORM_VALUES: UpdateSettingsInput = { ukvisajobsPassword: "", webhookSecret: "", enableBasicAuth: false, + backupEnabled: null, + backupHour: null, + backupMaxCount: null, }; type LlmProviderValue = LlmProviderId | null; @@ -107,6 +112,9 @@ const NULL_SETTINGS_PAYLOAD: UpdateSettingsInput = { ukvisajobsPassword: null, webhookSecret: null, enableBasicAuth: undefined, + backupEnabled: null, + backupHour: null, + backupMaxCount: null, }; const mapSettingsToForm = (data: AppSettings): UpdateSettingsInput => ({ @@ -141,6 +149,9 @@ const mapSettingsToForm = (data: AppSettings): UpdateSettingsInput => ({ ukvisajobsPassword: "", webhookSecret: "", enableBasicAuth: data.basicAuthActive, + backupEnabled: data.overrideBackupEnabled, + backupHour: data.overrideBackupHour, + backupMaxCount: data.overrideBackupMaxCount, }); const normalizeString = (value: string | null | undefined) => { @@ -307,6 +318,21 @@ const getDerivedSettings = (settings: AppSettings | null) => { profileProjects, maxProjectsTotal: profileProjects.length, + + backup: { + backupEnabled: { + effective: settings?.backupEnabled ?? false, + default: settings?.defaultBackupEnabled ?? false, + }, + backupHour: { + effective: settings?.backupHour ?? 2, + default: settings?.defaultBackupHour ?? 2, + }, + backupMaxCount: { + effective: settings?.backupMaxCount ?? 5, + default: settings?.defaultBackupMaxCount ?? 5, + }, + }, }; }; @@ -326,6 +352,13 @@ export const SettingsPage: React.FC = () => { const [isFetchingRxResumeProjects, setIsFetchingRxResumeProjects] = useState(false); + // Backup state + const [backups, setBackups] = useState([]); + const [nextScheduled, setNextScheduled] = useState(null); + const [isLoadingBackups, setIsLoadingBackups] = useState(false); + const [isCreatingBackup, setIsCreatingBackup] = useState(false); + const [isDeletingBackup, setIsDeletingBackup] = useState(false); + const methods = useForm({ resolver: zodResolver( updateSettingsSchema, @@ -446,8 +479,68 @@ export const SettingsPage: React.FC = () => { envSettings, defaultResumeProjects, profileProjects, + backup, } = derived; + // Backup functions + const loadBackups = useCallback(async () => { + setIsLoadingBackups(true); + try { + const response = await api.getBackups(); + setBackups(response.backups); + setNextScheduled(response.nextScheduled); + } catch (error) { + const message = + error instanceof Error ? error.message : "Failed to load backups"; + toast.error(message); + } finally { + setIsLoadingBackups(false); + } + }, []); + + const handleCreateBackup = async () => { + setIsCreatingBackup(true); + try { + await api.createManualBackup(); + toast.success("Backup created successfully"); + await loadBackups(); + } catch (error) { + const message = + error instanceof Error ? error.message : "Failed to create backup"; + toast.error(message); + } finally { + setIsCreatingBackup(false); + } + }; + + const handleDeleteBackup = async (filename: string) => { + const confirmed = window.confirm( + `Delete backup "${filename}"? This action cannot be undone.`, + ); + if (!confirmed) { + return; + } + setIsDeletingBackup(true); + try { + await api.deleteBackup(filename); + toast.success("Backup deleted successfully"); + await loadBackups(); + } catch (error) { + const message = + error instanceof Error ? error.message : "Failed to delete backup"; + toast.error(message); + } finally { + setIsDeletingBackup(false); + } + }; + + // Load backups when settings are loaded + useEffect(() => { + if (settings) { + loadBackups(); + } + }, [settings, loadBackups]); + const effectiveProfileProjects = rxResumeProjectsOverride ?? profileProjects; const effectiveMaxProjectsTotal = effectiveProfileProjects.length; @@ -589,6 +682,15 @@ export const SettingsPage: React.FC = () => { jobspy.isRemote.default, ), showSponsorInfo: nullIfSame(data.showSponsorInfo, display.default), + backupEnabled: nullIfSame( + data.backupEnabled, + backup.backupEnabled.default, + ), + backupHour: nullIfSame(data.backupHour, backup.backupHour.default), + backupMaxCount: nullIfSame( + data.backupMaxCount, + backup.backupMaxCount.default, + ), ...envPayload, }; @@ -751,6 +853,17 @@ export const SettingsPage: React.FC = () => { isLoading={isLoading} isSaving={isSaving} /> + void; + onDeleteBackup: (filename: string) => void; + isCreatingBackup: boolean; + isDeletingBackup: boolean; +}; + +const formatFileSize = (bytes: number): string => { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +}; + +const formatBackupDate = (dateString: string): string => { + const date = new Date(dateString); + return date.toLocaleString("en-US", { + timeZone: "UTC", + month: "short", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "2-digit", + hour12: true, + timeZoneName: "short", + }); +}; + +export const BackupSettingsSection: React.FC = ({ + values, + backups, + nextScheduled, + isLoading, + isSaving, + onCreateBackup, + onDeleteBackup, + isCreatingBackup, + isDeletingBackup, +}) => { + const { backupEnabled, backupHour, backupMaxCount } = values; + const { control, watch } = useFormContext(); + + // Watch the current form value to conditionally show/hide fields + const currentBackupEnabled = watch("backupEnabled") ?? backupEnabled.default; + + return ( + + + Backup + + +
+ {/* Enable automatic backups toggle */} +
+ ( + { + field.onChange( + checked === "indeterminate" ? null : checked === true, + ); + }} + disabled={isLoading || isSaving} + /> + )} + /> +
+ +

+ Automatically create database backups on a daily schedule. + Manual backups can always be created regardless of this setting. +

+
+
+ + {/* Backup settings - only shown when enabled */} + {currentBackupEnabled && ( +
+ ( + { + const value = parseInt(event.target.value, 10); + if (Number.isNaN(value)) { + field.onChange(null); + } else { + field.onChange(Math.min(23, Math.max(0, value))); + } + }, + }} + disabled={isLoading || isSaving} + helper={`Hour of the day (0-23) in UTC when automatic backups should run. Default: ${backupHour.default}:00 UTC.`} + current={`Effective: ${backupHour.effective}:00 UTC | Default: ${backupHour.default}:00 UTC`} + /> + )} + /> + + ( + { + const value = parseInt(event.target.value, 10); + if (Number.isNaN(value)) { + field.onChange(null); + } else { + field.onChange(Math.min(5, Math.max(1, value))); + } + }, + }} + disabled={isLoading || isSaving} + helper={`Maximum number of automatic backups to retain (1-5). Older backups are deleted automatically. Default: ${backupMaxCount.default}.`} + current={`Effective: ${backupMaxCount.effective} | Default: ${backupMaxCount.default}`} + /> + )} + /> +
+ )} + + {/* Next scheduled backup */} + {currentBackupEnabled && nextScheduled && ( +
+ + + Next scheduled backup: {formatBackupDate(nextScheduled)} + +
+ )} + + + + {/* Backup list */} +
+
+
Backup History
+ +
+ + + + {backups.length} backup{backups.length !== 1 ? "s" : ""} + +
+ } + > + {backups.length === 0 ? ( + + ) : ( + backups.map((backup) => ( + +
+ +
+
+ {backup.filename} +
+
+ {formatBackupDate(backup.createdAt)} ·{" "} + {formatFileSize(backup.size)} +
+
+
+
+ + {backup.type === "auto" ? "Auto" : "Manual"} + + +
+ + )) + )} + +
+ + + + {/* Effective/Default values display */} +
+
+
Enabled
+
+ Effective: {backupEnabled.effective ? "Yes" : "No"} | Default:{" "} + {backupEnabled.default ? "Yes" : "No"} +
+
+
+
Hour
+
+ Effective: {backupHour.effective}:00 UTC | Default:{" "} + {backupHour.default}:00 UTC +
+
+
+
Max Count
+
+ Effective: {backupMaxCount.effective} | Default:{" "} + {backupMaxCount.default} +
+
+
+ + + + ); +}; diff --git a/orchestrator/src/client/pages/settings/types.ts b/orchestrator/src/client/pages/settings/types.ts index d4cf39d..d1f278d 100644 --- a/orchestrator/src/client/pages/settings/types.ts +++ b/orchestrator/src/client/pages/settings/types.ts @@ -42,3 +42,9 @@ export type EnvSettingsValues = { }; basicAuthActive: boolean; }; + +export type BackupValues = { + backupEnabled: EffectiveDefault; + backupHour: EffectiveDefault; + backupMaxCount: EffectiveDefault; +}; diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts index be7d0a6..44ffc55 100644 --- a/orchestrator/src/server/api/routes.ts +++ b/orchestrator/src/server/api/routes.ts @@ -3,6 +3,7 @@ */ import { Router } from "express"; +import { backupRouter } from "./routes/backup.js"; import { databaseRouter } from "./routes/database.js"; import { jobsRouter } from "./routes/jobs.js"; import { manualJobsRouter } from "./routes/manual-jobs.js"; @@ -26,3 +27,4 @@ apiRouter.use("/profile", profileRouter); apiRouter.use("/database", databaseRouter); apiRouter.use("/visa-sponsors", visaSponsorsRouter); apiRouter.use("/onboarding", onboardingRouter); +apiRouter.use("/backups", backupRouter); diff --git a/orchestrator/src/server/api/routes/backup.test.ts b/orchestrator/src/server/api/routes/backup.test.ts new file mode 100644 index 0000000..33f2660 --- /dev/null +++ b/orchestrator/src/server/api/routes/backup.test.ts @@ -0,0 +1,122 @@ +import fs from "node:fs"; +import type { Server } from "node:http"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { startServer, stopServer } from "./test-utils.js"; + +describe.sequential("Backup API routes", () => { + let server: Server; + let baseUrl: string; + let closeDb: () => void; + let tempDir: string; + + beforeEach(async () => { + ({ server, baseUrl, closeDb, tempDir } = await startServer()); + }); + + afterEach(async () => { + await stopServer({ server, closeDb, tempDir }); + }); + + describe("GET /api/backups", () => { + it("should return empty array when no backups exist", async () => { + const res = await fetch(`${baseUrl}/api/backups`); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body.success).toBe(true); + expect(body.data.backups).toEqual([]); + expect(body.data.nextScheduled).toBeNull(); + }); + + it("should list backups with metadata", async () => { + // Create a backup first + await fetch(`${baseUrl}/api/backups`, { method: "POST" }); + + const res = await fetch(`${baseUrl}/api/backups`); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body.success).toBe(true); + expect(body.data.backups).toHaveLength(1); + expect(body.data.backups[0]).toHaveProperty("filename"); + expect(body.data.backups[0]).toHaveProperty("type", "manual"); + expect(body.data.backups[0]).toHaveProperty("size"); + expect(body.data.backups[0]).toHaveProperty("createdAt"); + }); + }); + + describe("POST /api/backups", () => { + it("should create a manual backup", async () => { + const res = await fetch(`${baseUrl}/api/backups`, { method: "POST" }); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body.success).toBe(true); + expect(body.data.type).toBe("manual"); + expect(body.data.filename).toMatch( + /^jobs_manual_\d{4}_\d{2}_\d{2}_\d{2}_\d{2}_\d{2}\.db$/, + ); + expect(body.data.size).toBeGreaterThan(0); + }); + + it("should return error if database does not exist", async () => { + // Delete the database + await fs.promises.unlink(`${tempDir}/jobs.db`); + + const res = await fetch(`${baseUrl}/api/backups`, { method: "POST" }); + const body = await res.json(); + + expect(res.status).toBe(500); + expect(body.success).toBe(false); + expect(body.error).toContain("Database file not found"); + }); + }); + + describe("DELETE /api/backups/:filename", () => { + it("should delete a backup", async () => { + // Create a backup first + const createRes = await fetch(`${baseUrl}/api/backups`, { + method: "POST", + }); + const createBody = await createRes.json(); + const filename = createBody.data.filename; + + // Delete the backup + const deleteRes = await fetch(`${baseUrl}/api/backups/${filename}`, { + method: "DELETE", + }); + const deleteBody = await deleteRes.json(); + + expect(deleteRes.status).toBe(200); + expect(deleteBody.success).toBe(true); + expect(deleteBody.message).toContain("deleted successfully"); + + // Verify it's gone + const listRes = await fetch(`${baseUrl}/api/backups`); + const listBody = await listRes.json(); + expect(listBody.data.backups).toHaveLength(0); + }); + + it("should return 404 for non-existent backup", async () => { + const res = await fetch(`${baseUrl}/api/backups/jobs_2026_01_01.db`, { + method: "DELETE", + }); + const body = await res.json(); + + expect(res.status).toBe(404); + expect(body.success).toBe(false); + expect(body.error).toContain("not found"); + }); + + it("should return 400 for invalid filename", async () => { + const res = await fetch(`${baseUrl}/api/backups/invalid_filename.txt`, { + method: "DELETE", + }); + const body = await res.json(); + + expect(res.status).toBe(400); + expect(body.success).toBe(false); + expect(body.error).toContain("Invalid"); + }); + }); +}); diff --git a/orchestrator/src/server/api/routes/backup.ts b/orchestrator/src/server/api/routes/backup.ts new file mode 100644 index 0000000..ddb1b56 --- /dev/null +++ b/orchestrator/src/server/api/routes/backup.ts @@ -0,0 +1,93 @@ +import { + createBackup, + deleteBackup, + getNextBackupTime, + listBackups, +} from "@server/services/backup/index.js"; +import { type Request, type Response, Router } from "express"; + +export const backupRouter = Router(); + +/** + * GET /api/backups - List all backups with metadata + */ +backupRouter.get("/", async (_req: Request, res: Response) => { + try { + const backups = await listBackups(); + const nextScheduled = getNextBackupTime(); + + res.json({ + success: true, + data: { + backups, + nextScheduled, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown error"; + console.error("❌ [backup-api] Failed to list backups:", error); + res.status(500).json({ success: false, error: message }); + } +}); + +/** + * POST /api/backups - Create a manual backup + */ +backupRouter.post("/", async (_req: Request, res: Response) => { + try { + const filename = await createBackup("manual"); + const backups = await listBackups(); + const backup = backups.find((b) => b.filename === filename); + + if (!backup) { + throw new Error("Backup was created but not found in list"); + } + + res.json({ + success: true, + data: backup, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown error"; + console.error("❌ [backup-api] Failed to create backup:", error); + res.status(500).json({ success: false, error: message }); + } +}); + +/** + * DELETE /api/backups/:filename - Delete a specific backup + */ +backupRouter.delete("/:filename", async (req: Request, res: Response) => { + try { + const { filename } = req.params; + + if (!filename) { + res.status(400).json({ + success: false, + error: "Filename is required", + }); + return; + } + + await deleteBackup(filename); + + res.json({ + success: true, + message: `Backup ${filename} deleted successfully`, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown error"; + console.error( + `❌ [backup-api] Failed to delete backup ${req.params.filename}:`, + error, + ); + + if (message.includes("not found")) { + res.status(404).json({ success: false, error: message }); + } else if (message.includes("Invalid")) { + res.status(400).json({ success: false, error: message }); + } else { + res.status(500).json({ success: false, error: message }); + } + } +}); diff --git a/orchestrator/src/server/api/routes/settings.ts b/orchestrator/src/server/api/routes/settings.ts index dd9638d..19c22a8 100644 --- a/orchestrator/src/server/api/routes/settings.ts +++ b/orchestrator/src/server/api/routes/settings.ts @@ -1,4 +1,5 @@ import * as settingsRepo from "@server/repositories/settings.js"; +import { setBackupSettings } from "@server/services/backup/index.js"; import { applyEnvValue, normalizeEnvInput, @@ -328,9 +329,53 @@ settingsRouter.patch("/", async (req: Request, res: Response) => { ); } + // Backup settings + if ("backupEnabled" in input) { + const val = input.backupEnabled ?? null; + promises.push( + settingsRepo.setSetting( + "backupEnabled", + val !== null ? (val ? "1" : "0") : null, + ), + ); + } + + if ("backupHour" in input) { + const val = input.backupHour ?? null; + promises.push( + settingsRepo.setSetting( + "backupHour", + val !== null ? String(val) : null, + ), + ); + } + + if ("backupMaxCount" in input) { + const val = input.backupMaxCount ?? null; + promises.push( + settingsRepo.setSetting( + "backupMaxCount", + val !== null ? String(val) : null, + ), + ); + } + await Promise.all(promises); const data = await getEffectiveSettings(); + + // Update backup scheduler if backup settings changed + if ( + "backupEnabled" in input || + "backupHour" in input || + "backupMaxCount" in input + ) { + setBackupSettings({ + enabled: data.backupEnabled, + hour: data.backupHour, + maxCount: data.backupMaxCount, + }); + } res.json({ success: true, data }); } catch (error) { const message = error instanceof Error ? error.message : "Unknown error"; diff --git a/orchestrator/src/server/index.ts b/orchestrator/src/server/index.ts index 4f510c5..d44e7ab 100644 --- a/orchestrator/src/server/index.ts +++ b/orchestrator/src/server/index.ts @@ -4,6 +4,12 @@ import "./config/env.js"; import { createApp } from "./app.js"; +import * as settingsRepo from "./repositories/settings.js"; +import { + getBackupSettings, + setBackupSettings, + startBackupScheduler, +} from "./services/backup/index.js"; import { applyStoredEnvOverrides } from "./services/envSettings.js"; import { initialize as initializeVisaSponsors } from "./services/visa-sponsors/index.js"; @@ -35,6 +41,45 @@ async function startServer() { } catch (error) { console.warn("⚠️ Failed to initialize visa sponsors service:", error); } + + // Initialize backup service (load settings and start scheduler if enabled) + try { + const backupEnabled = await settingsRepo.getSetting("backupEnabled"); + const backupHour = await settingsRepo.getSetting("backupHour"); + const backupMaxCount = await settingsRepo.getSetting("backupMaxCount"); + + const parsedHour = backupHour ? parseInt(backupHour, 10) : NaN; + const parsedMaxCount = backupMaxCount + ? parseInt(backupMaxCount, 10) + : NaN; + const safeHour = Number.isNaN(parsedHour) + ? 2 + : Math.min(23, Math.max(0, parsedHour)); + const safeMaxCount = Number.isNaN(parsedMaxCount) + ? 5 + : Math.min(5, Math.max(1, parsedMaxCount)); + + setBackupSettings({ + enabled: backupEnabled === "true" || backupEnabled === "1", + hour: safeHour, + maxCount: safeMaxCount, + }); + + startBackupScheduler(); + + const settings = getBackupSettings(); + if (settings.enabled) { + console.log( + `✅ Backup scheduler started (hour: ${settings.hour}, max: ${settings.maxCount})`, + ); + } else { + console.log( + "ℹ️ Backups disabled. Enable in settings to schedule automatic backups.", + ); + } + } catch (error) { + console.warn("⚠️ Failed to initialize backup service:", error); + } }); } diff --git a/orchestrator/src/server/repositories/settings.ts b/orchestrator/src/server/repositories/settings.ts index b35778f..4ae8cda 100644 --- a/orchestrator/src/server/repositories/settings.ts +++ b/orchestrator/src/server/repositories/settings.ts @@ -37,7 +37,10 @@ export type SettingKey = | "basicAuthPassword" | "ukvisajobsEmail" | "ukvisajobsPassword" - | "webhookSecret"; + | "webhookSecret" + | "backupEnabled" + | "backupHour" + | "backupMaxCount"; export async function getSetting(key: SettingKey): Promise { const [row] = await db.select().from(settings).where(eq(settings.key, key)); diff --git a/orchestrator/src/server/services/backup/index.test.ts b/orchestrator/src/server/services/backup/index.test.ts new file mode 100644 index 0000000..fbcdc34 --- /dev/null +++ b/orchestrator/src/server/services/backup/index.test.ts @@ -0,0 +1,366 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import Database from "better-sqlite3"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as backup from "./index.js"; + +// Mock the dataDir module +vi.mock("../../config/dataDir.js", () => ({ + getDataDir: vi.fn(), +})); + +import { getDataDir } from "../../config/dataDir.js"; + +describe("Backup Service", () => { + let tempDir: string; + let dbPath: string; + + beforeEach(async () => { + // Create temp directory for tests + tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "backup-test-")); + dbPath = path.join(tempDir, "jobs.db"); + + // Create a real SQLite database file for backup() to work. + const db = new Database(dbPath); + try { + db.exec( + [ + "PRAGMA journal_mode = DELETE;", + "CREATE TABLE IF NOT EXISTS test_items (id INTEGER PRIMARY KEY, name TEXT NOT NULL);", + "DELETE FROM test_items;", + "INSERT INTO test_items (name) VALUES ('alpha');", + ].join("\n"), + ); + } finally { + db.close(); + } + + // Mock getDataDir to return temp directory + vi.mocked(getDataDir).mockReturnValue(tempDir); + + // Reset backup settings + backup.setBackupSettings({ enabled: false, hour: 2, maxCount: 5 }); + backup.stopBackupScheduler(); + }); + + afterEach(async () => { + // Clean up temp directory + try { + await fs.promises.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + vi.clearAllMocks(); + }); + + describe("createBackup", () => { + it("should create an automatic backup with correct filename format", async () => { + const filename = await backup.createBackup("auto"); + + // Check filename format: jobs_YYYY_MM_DD.db + expect(filename).toMatch(/^jobs_\d{4}_\d{2}_\d{2}\.db$/); + + // Check file was created + const backupPath = path.join(tempDir, filename); + expect(fs.existsSync(backupPath)).toBe(true); + + // Check backup is a valid SQLite database with expected data + const backupDb = new Database(backupPath, { + readonly: true, + fileMustExist: true, + }); + try { + const row = backupDb + .prepare("SELECT name FROM test_items ORDER BY id LIMIT 1") + .get() as { name: string } | undefined; + expect(row?.name).toBe("alpha"); + } finally { + backupDb.close(); + } + }); + + it("should create a manual backup with correct filename format", async () => { + const filename = await backup.createBackup("manual"); + + // Check filename format: jobs_manual_YYYY_MM_DD_HH_MM_SS.db + expect(filename).toMatch( + /^jobs_manual_\d{4}_\d{2}_\d{2}_\d{2}_\d{2}_\d{2}\.db$/, + ); + + // Check file was created + const backupPath = path.join(tempDir, filename); + expect(fs.existsSync(backupPath)).toBe(true); + + const backupDb = new Database(backupPath, { + readonly: true, + fileMustExist: true, + }); + try { + const count = backupDb + .prepare("SELECT COUNT(*) as count FROM test_items") + .get() as { count: number }; + expect(count.count).toBe(1); + } finally { + backupDb.close(); + } + }); + + it("should add a suffix when manual backup name collides", async () => { + // Only fake Date to keep async I/O (used by better-sqlite3 backup) real. + vi.useFakeTimers({ toFake: ["Date"] }); + try { + vi.setSystemTime(new Date("2026-01-15T12:30:45Z")); + + const first = await backup.createBackup("manual"); + const second = await backup.createBackup("manual"); + + expect(first).toBe("jobs_manual_2026_01_15_12_30_45.db"); + expect(second).toBe("jobs_manual_2026_01_15_12_30_45_1.db"); + expect(fs.existsSync(path.join(tempDir, second))).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it("should throw error if database does not exist", async () => { + // Delete the database + await fs.promises.unlink(dbPath); + + await expect(backup.createBackup("auto")).rejects.toThrow( + "Database file not found", + ); + }); + }); + + describe("listBackups", () => { + it("should return empty array when no backups exist", async () => { + const backups = await backup.listBackups(); + expect(backups).toEqual([]); + }); + + it("should list all backups with metadata", async () => { + // Create some backups + await backup.createBackup("auto"); + await backup.createBackup("manual"); + + const backups = await backup.listBackups(); + + expect(backups).toHaveLength(2); + expect(backups[0]).toHaveProperty("filename"); + expect(backups[0]).toHaveProperty("type"); + expect(backups[0]).toHaveProperty("size"); + expect(backups[0]).toHaveProperty("createdAt"); + }); + + it("should sort backups by date (newest first)", async () => { + // Create backups with different dates by manipulating filenames + const oldBackup = path.join(tempDir, "jobs_2026_01_01.db"); + const newBackup = path.join(tempDir, "jobs_2026_01_15.db"); + await fs.promises.writeFile(oldBackup, "old"); + await fs.promises.writeFile(newBackup, "new"); + + const backups = await backup.listBackups(); + + expect(backups[0].filename).toBe("jobs_2026_01_15.db"); + expect(backups[1].filename).toBe("jobs_2026_01_01.db"); + }); + + it("should ignore non-backup files", async () => { + // Create a backup and some other files + await backup.createBackup("auto"); + await fs.promises.writeFile(path.join(tempDir, "random.txt"), "text"); + await fs.promises.writeFile(path.join(tempDir, "jobs.db"), "db"); + + const backups = await backup.listBackups(); + + expect(backups).toHaveLength(1); + expect(backups[0].filename).toMatch(/^jobs_\d{4}_\d{2}_\d{2}\.db$/); + }); + + it("should include suffixed manual backups", async () => { + const filename = "jobs_manual_2026_01_01_12_00_00_2.db"; + await fs.promises.writeFile(path.join(tempDir, filename), "manual"); + + const backups = await backup.listBackups(); + + expect(backups).toHaveLength(1); + expect(backups[0].filename).toBe(filename); + expect(backups[0].type).toBe("manual"); + expect(backups[0].createdAt).toBe("2026-01-01T12:00:00.000Z"); + }); + }); + + describe("deleteBackup", () => { + it("should delete a backup file", async () => { + const filename = await backup.createBackup("auto"); + const backupPath = path.join(tempDir, filename); + + expect(fs.existsSync(backupPath)).toBe(true); + + await backup.deleteBackup(filename); + + expect(fs.existsSync(backupPath)).toBe(false); + }); + + it("should throw error for invalid filename", async () => { + await expect(backup.deleteBackup("../../../etc/passwd")).rejects.toThrow( + "Invalid backup filename", + ); + await expect(backup.deleteBackup("random.txt")).rejects.toThrow( + "Invalid backup filename", + ); + }); + + it("should throw error if backup does not exist", async () => { + await expect(backup.deleteBackup("jobs_2026_01_01.db")).rejects.toThrow( + "Backup not found", + ); + }); + + it("should delete a suffixed manual backup", async () => { + const filename = "jobs_manual_2026_01_01_12_00_00_1.db"; + await fs.promises.writeFile(path.join(tempDir, filename), "manual"); + + await backup.deleteBackup(filename); + + expect(fs.existsSync(path.join(tempDir, filename))).toBe(false); + }); + }); + + describe("cleanupOldBackups", () => { + it("should delete oldest automatic backups when exceeding max count", async () => { + // Create 7 auto backups (max is 5) + for (let i = 1; i <= 7; i++) { + const filename = `jobs_2026_01_${String(i).padStart(2, "0")}.db`; + await fs.promises.writeFile(path.join(tempDir, filename), "data"); + } + + // Set max count to 5 + backup.setBackupSettings({ maxCount: 5 }); + await backup.cleanupOldBackups(); + + const remaining = await backup.listBackups(); + expect(remaining).toHaveLength(5); + // Should keep the 5 newest (03-07) + const filenames = remaining.map((b) => b.filename); + expect(filenames).toContain("jobs_2026_01_03.db"); + expect(filenames).toContain("jobs_2026_01_07.db"); + expect(filenames).not.toContain("jobs_2026_01_01.db"); + expect(filenames).not.toContain("jobs_2026_01_02.db"); + }); + + it("should not delete manual backups", async () => { + // Create auto backups + for (let i = 1; i <= 7; i++) { + const filename = `jobs_2026_01_${String(i).padStart(2, "0")}.db`; + await fs.promises.writeFile(path.join(tempDir, filename), "data"); + } + + // Create manual backups + await fs.promises.writeFile( + path.join(tempDir, "jobs_manual_2026_01_01_12_00_00.db"), + "manual", + ); + await fs.promises.writeFile( + path.join(tempDir, "jobs_manual_2026_01_02_12_00_00.db"), + "manual", + ); + + backup.setBackupSettings({ maxCount: 5 }); + await backup.cleanupOldBackups(); + + const remaining = await backup.listBackups(); + const autoBackups = remaining.filter((b) => b.type === "auto"); + const manualBackups = remaining.filter((b) => b.type === "manual"); + + expect(autoBackups).toHaveLength(5); + expect(manualBackups).toHaveLength(2); + }); + + it("should not delete if count is within limit", async () => { + // Create 3 auto backups (max is 5) + for (let i = 1; i <= 3; i++) { + const filename = `jobs_2026_01_${String(i).padStart(2, "0")}.db`; + await fs.promises.writeFile(path.join(tempDir, filename), "data"); + } + + backup.setBackupSettings({ maxCount: 5 }); + await backup.cleanupOldBackups(); + + const remaining = await backup.listBackups(); + expect(remaining).toHaveLength(3); + }); + }); + + describe("setBackupSettings", () => { + it("should update settings", () => { + backup.setBackupSettings({ enabled: true, hour: 4, maxCount: 3 }); + + const settings = backup.getBackupSettings(); + expect(settings.enabled).toBe(true); + expect(settings.hour).toBe(4); + expect(settings.maxCount).toBe(3); + }); + + it("should merge partial settings", () => { + backup.setBackupSettings({ hour: 6 }); + + const settings = backup.getBackupSettings(); + expect(settings.enabled).toBe(false); // unchanged + expect(settings.hour).toBe(6); // updated + expect(settings.maxCount).toBe(5); // unchanged + }); + }); + + describe("scheduler integration", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should start scheduler when enabled", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + expect(backup.isBackupSchedulerRunning()).toBe(false); + + backup.setBackupSettings({ enabled: true, hour: 14 }); + + expect(backup.isBackupSchedulerRunning()).toBe(true); + expect(backup.getNextBackupTime()).not.toBeNull(); + }); + + it("should stop scheduler when disabled", () => { + backup.setBackupSettings({ enabled: true, hour: 14 }); + expect(backup.isBackupSchedulerRunning()).toBe(true); + + backup.setBackupSettings({ enabled: false }); + expect(backup.isBackupSchedulerRunning()).toBe(false); + expect(backup.getNextBackupTime()).toBeNull(); + }); + + it("should restart scheduler when hour changes", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + backup.setBackupSettings({ enabled: true, hour: 14 }); + const firstRun = backup.getNextBackupTime(); + + backup.setBackupSettings({ hour: 16 }); + const secondRun = backup.getNextBackupTime(); + + expect(secondRun).not.toBe(firstRun); + expect(secondRun).not.toBeNull(); + expect(firstRun).not.toBeNull(); + if (secondRun && firstRun) { + expect(new Date(secondRun).getTime()).toBeGreaterThan( + new Date(firstRun).getTime(), + ); + } + }); + }); +}); diff --git a/orchestrator/src/server/services/backup/index.ts b/orchestrator/src/server/services/backup/index.ts new file mode 100644 index 0000000..f5e2cfc --- /dev/null +++ b/orchestrator/src/server/services/backup/index.ts @@ -0,0 +1,398 @@ +/** + * Database Backup Service + * + * Manages automatic and manual backups of the SQLite database. + * Stores backups in the same directory as the original database. + */ + +import fs from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import path from "node:path"; +import type { BackupInfo } from "@shared/types.js"; +import Database from "better-sqlite3"; +import { getDataDir } from "../../config/dataDir.js"; +import { createScheduler } from "../../utils/scheduler.js"; + +const DB_FILENAME = "jobs.db"; +const AUTO_BACKUP_PREFIX = "jobs_"; +const MANUAL_BACKUP_PREFIX = "jobs_manual_"; +const AUTO_BACKUP_PATTERN = /^jobs_\d{4}_\d{2}_\d{2}\.db$/; +const MANUAL_BACKUP_PATTERN = + /^jobs_manual_\d{4}_\d{2}_\d{2}_\d{2}_\d{2}_\d{2}(?:_\d+)?\.db$/; + +const AUTO_BACKUP_REGEX = /^jobs_(\d{4})_(\d{2})_(\d{2})\.db$/; +const MANUAL_BACKUP_REGEX = + /^jobs_manual_(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})(?:_\d+)?\.db$/; + +type SqliteDatabase = InstanceType; + +interface BackupSettings { + enabled: boolean; + hour: number; + maxCount: number; +} + +// Current settings (updated by setBackupSettings) +let currentSettings: BackupSettings = { + enabled: false, + hour: 2, + maxCount: 5, +}; + +// Create scheduler for automatic backups +const scheduler = createScheduler("backup", async () => { + await createBackup("auto"); + await cleanupOldBackups(); +}); + +/** + * Get the path to the database file + */ +function getDbPath(): string { + return path.join(getDataDir(), DB_FILENAME); +} + +/** + * Get the data directory path + */ +function getBackupDir(): string { + return getDataDir(); +} + +/** + * Generate filename for a backup + */ +function generateBackupFilename(type: "auto" | "manual"): string { + const now = new Date(); + if (type === "auto") { + // Format: jobs_YYYY_MM_DD.db (UTC date to match UTC scheduler) + const year = now.getUTCFullYear(); + const month = String(now.getUTCMonth() + 1).padStart(2, "0"); + const day = String(now.getUTCDate()).padStart(2, "0"); + return `${AUTO_BACKUP_PREFIX}${year}_${month}_${day}.db`; + } else { + // Format: jobs_manual_YYYY_MM_DD_HH_MM_SS.db (local time for manual backups) + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + const hours = String(now.getHours()).padStart(2, "0"); + const minutes = String(now.getMinutes()).padStart(2, "0"); + const seconds = String(now.getSeconds()).padStart(2, "0"); + return `${MANUAL_BACKUP_PREFIX}${year}_${month}_${day}_${hours}_${minutes}_${seconds}.db`; + } +} + +/** + * Parse backup filename to extract creation date + */ +function parseBackupDate(filename: string): Date | null { + const autoMatch = filename.match(AUTO_BACKUP_REGEX); + if (autoMatch) { + const [, year, month, day] = autoMatch; + return buildUtcDate(year, month, day, "0", "0", "0"); + } + + const manualMatch = filename.match(MANUAL_BACKUP_REGEX); + if (manualMatch) { + const [, year, month, day, hours, minutes, seconds] = manualMatch; + return buildUtcDate(year, month, day, hours, minutes, seconds); + } + + return null; +} + +function buildUtcDate( + yearRaw: string, + monthRaw: string, + dayRaw: string, + hourRaw: string, + minuteRaw: string, + secondRaw: string, +): Date | null { + const year = Number(yearRaw); + const month = Number(monthRaw); + const day = Number(dayRaw); + const hour = Number(hourRaw); + const minute = Number(minuteRaw); + const second = Number(secondRaw); + + const date = new Date(Date.UTC(year, month - 1, day, hour, minute, second)); + if (Number.isNaN(date.getTime())) { + return null; + } + + if ( + date.getUTCFullYear() !== year || + date.getUTCMonth() + 1 !== month || + date.getUTCDate() !== day || + date.getUTCHours() !== hour || + date.getUTCMinutes() !== minute || + date.getUTCSeconds() !== second + ) { + return null; + } + + return date; +} + +/** + * Determine backup type from filename + */ +function getBackupType(filename: string): "auto" | "manual" | null { + if (AUTO_BACKUP_PATTERN.test(filename)) return "auto"; + if (MANUAL_BACKUP_PATTERN.test(filename)) return "manual"; + return null; +} + +/** + * Create a backup of the database + * @param type - 'auto' for scheduled backups, 'manual' for user-triggered + * @returns The filename of the created backup + */ +export async function createBackup(type: "auto" | "manual"): Promise { + const dbPath = getDbPath(); + const backupDir = getBackupDir(); + const baseFilename = generateBackupFilename(type); + let filename = baseFilename; + let backupPath = path.join(backupDir, filename); + let reservedHandle: FileHandle | null = null; + + // Check if database exists + if (!fs.existsSync(dbPath)) { + throw new Error(`Database file not found: ${dbPath}`); + } + + const tryReserve = async ( + candidatePath: string, + ): Promise => { + try { + return await fs.promises.open(candidatePath, "wx"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return null; + throw error; + } + }; + + if (type === "auto") { + reservedHandle = await tryReserve(backupPath); + if (!reservedHandle) { + console.log( + `ℹ️ [backup] Auto backup already exists for today: ${filename}`, + ); + return filename; + } + } else { + const baseName = baseFilename.replace(/\.db$/, ""); + let sequence = 0; + + while (!reservedHandle && sequence <= 100) { + const candidate = + sequence === 0 ? baseFilename : `${baseName}_${sequence}.db`; + const candidatePath = path.join(backupDir, candidate); + const reserved = await tryReserve(candidatePath); + if (reserved) { + reservedHandle = reserved; + filename = candidate; + backupPath = candidatePath; + } else { + sequence += 1; + } + } + + if (!reservedHandle) { + throw new Error("Failed to create unique manual backup filename"); + } + } + + // Close the reserved file handle before running SQLite backup + await reservedHandle.close(); + + let sqlite: SqliteDatabase | null = null; + try { + sqlite = new Database(dbPath, { readonly: true, fileMustExist: true }); + await sqlite.backup(backupPath); + } catch (error) { + await fs.promises.unlink(backupPath).catch(() => undefined); + throw error; + } finally { + sqlite?.close(); + } + + console.log( + `✅ [backup] Created ${type} backup: ${filename} (${(await fs.promises.stat(backupPath)).size} bytes)`, + ); + + return filename; +} + +/** + * List all backups with metadata + * @returns Array of backup information + */ +export async function listBackups(): Promise { + const backupDir = getBackupDir(); + + // Check if directory exists + if (!fs.existsSync(backupDir)) { + return []; + } + + // Read directory and filter backup files + const files = await fs.promises.readdir(backupDir); + const backupFiles = files.filter((file) => { + return AUTO_BACKUP_PATTERN.test(file) || MANUAL_BACKUP_PATTERN.test(file); + }); + + // Get metadata for each backup + const backups: BackupInfo[] = []; + for (const filename of backupFiles) { + const filePath = path.join(backupDir, filename); + const type = getBackupType(filename); + const createdAt = parseBackupDate(filename); + + if (type && createdAt) { + const stats = await fs.promises.stat(filePath); + backups.push({ + filename, + type, + size: stats.size, + createdAt: createdAt.toISOString(), + }); + } + } + + // Sort by creation date (newest first) + backups.sort((a, b) => { + return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); + }); + + return backups; +} + +/** + * Delete a specific backup + * @param filename - Name of the backup file to delete + */ +export async function deleteBackup(filename: string): Promise { + // Validate filename to prevent path traversal + if ( + !AUTO_BACKUP_PATTERN.test(filename) && + !MANUAL_BACKUP_PATTERN.test(filename) + ) { + throw new Error("Invalid backup filename"); + } + + const backupDir = getBackupDir(); + const filePath = path.join(backupDir, filename); + + // Check if file exists + if (!fs.existsSync(filePath)) { + throw new Error(`Backup not found: ${filename}`); + } + + // Delete file + await fs.promises.unlink(filePath); + console.log(`🗑️ [backup] Deleted backup: ${filename}`); +} + +/** + * Clean up old automatic backups + * Keeps only the most recent N automatic backups (where N = maxCount) + * Manual backups are never deleted automatically + */ +export async function cleanupOldBackups(): Promise { + const backups = await listBackups(); + + // Filter to only automatic backups + const autoBackups = backups.filter((b) => b.type === "auto"); + + // Sort by creation date (oldest first for deletion) + autoBackups.sort((a, b) => { + return new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(); + }); + + // Delete oldest backups if we exceed max count + const maxCount = currentSettings.maxCount; + if (autoBackups.length > maxCount) { + const toDelete = autoBackups.slice(0, autoBackups.length - maxCount); + + for (const backup of toDelete) { + try { + await deleteBackup(backup.filename); + } catch (error) { + console.error( + `❌ [backup] Failed to delete old backup ${backup.filename}:`, + error, + ); + } + } + + console.log( + `🧹 [backup] Cleaned up ${toDelete.length} old automatic backups (max: ${maxCount})`, + ); + } +} + +/** + * Update backup settings and restart scheduler if needed + * @param settings - New backup settings + */ +export function setBackupSettings(settings: Partial): void { + const oldEnabled = currentSettings.enabled; + const oldHour = currentSettings.hour; + + // Update settings + currentSettings = { ...currentSettings, ...settings }; + + console.log(`⚙️ [backup] Settings updated:`, currentSettings); + + // Restart scheduler if settings changed + if (currentSettings.enabled) { + if (!oldEnabled || oldHour !== currentSettings.hour) { + // Start or restart with new hour + scheduler.start(currentSettings.hour); + } + } else if (oldEnabled && !currentSettings.enabled) { + // Stop scheduler + scheduler.stop(); + } +} + +/** + * Get current backup settings + */ +export function getBackupSettings(): BackupSettings { + return { ...currentSettings }; +} + +/** + * Get the next scheduled backup time + * @returns ISO string of next backup time, or null if disabled + */ +export function getNextBackupTime(): string | null { + return scheduler.getNextRun(); +} + +/** + * Check if automatic backup scheduler is running + */ +export function isBackupSchedulerRunning(): boolean { + return scheduler.isRunning(); +} + +/** + * Start the backup scheduler manually (used on server startup) + * Only starts if backup is enabled + */ +export function startBackupScheduler(): void { + if (currentSettings.enabled) { + scheduler.start(currentSettings.hour); + } +} + +/** + * Stop the backup scheduler + */ +export function stopBackupScheduler(): void { + scheduler.stop(); +} diff --git a/orchestrator/src/server/services/settings.ts b/orchestrator/src/server/services/settings.ts index 2d9b973..6f44c57 100644 --- a/orchestrator/src/server/services/settings.ts +++ b/orchestrator/src/server/services/settings.ts @@ -184,6 +184,34 @@ export async function getEffectiveSettings(): Promise { : null; const showSponsorInfo = overrideShowSponsorInfo ?? defaultShowSponsorInfo; + // Backup settings + const defaultBackupEnabled = false; + const overrideBackupEnabledRaw = overrides.backupEnabled; + const overrideBackupEnabled = overrideBackupEnabledRaw + ? overrideBackupEnabledRaw === "true" || overrideBackupEnabledRaw === "1" + : null; + const backupEnabled = overrideBackupEnabled ?? defaultBackupEnabled; + + const defaultBackupHour = 2; + const overrideBackupHourRaw = overrides.backupHour; + const parsedBackupHour = overrideBackupHourRaw + ? parseInt(overrideBackupHourRaw, 10) + : NaN; + const overrideBackupHour = Number.isNaN(parsedBackupHour) + ? null + : Math.min(23, Math.max(0, parsedBackupHour)); + const backupHour = overrideBackupHour ?? defaultBackupHour; + + const defaultBackupMaxCount = 5; + const overrideBackupMaxCountRaw = overrides.backupMaxCount; + const parsedBackupMaxCount = overrideBackupMaxCountRaw + ? parseInt(overrideBackupMaxCountRaw, 10) + : NaN; + const overrideBackupMaxCount = Number.isNaN(parsedBackupMaxCount) + ? null + : Math.min(5, Math.max(1, parsedBackupMaxCount)); + const backupMaxCount = overrideBackupMaxCount ?? defaultBackupMaxCount; + return { ...envSettings, model, @@ -242,6 +270,15 @@ export async function getEffectiveSettings(): Promise { showSponsorInfo, defaultShowSponsorInfo, overrideShowSponsorInfo, + backupEnabled, + defaultBackupEnabled, + overrideBackupEnabled, + backupHour, + defaultBackupHour, + overrideBackupHour, + backupMaxCount, + defaultBackupMaxCount, + overrideBackupMaxCount, } as AppSettings; } diff --git a/orchestrator/src/server/services/visa-sponsors/index.ts b/orchestrator/src/server/services/visa-sponsors/index.ts index c6ea62f..5c1263a 100644 --- a/orchestrator/src/server/services/visa-sponsors/index.ts +++ b/orchestrator/src/server/services/visa-sponsors/index.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import path from "node:path"; import { getDataDir } from "../../config/dataDir.js"; +import { createScheduler } from "../../utils/scheduler.js"; const DATA_DIR = path.join(getDataDir(), "visa-sponsors"); @@ -492,75 +493,32 @@ export function getOrganizationDetails( } // ============================================================================ -// Scheduled Updates (Cron-style) +// Scheduled Updates (Cron-style) - Uses shared scheduler utility // ============================================================================ -let scheduledTimer: ReturnType | null = null; -let nextScheduledUpdateTime: Date | null = null; - -/** - * Calculate the next update time (default: 2 AM daily) - */ -function calculateNextUpdateTime(hour = 2): Date { - const now = new Date(); - const next = new Date(now); - next.setHours(hour, 0, 0, 0); - - // If we've passed the time today, schedule for tomorrow - if (next <= now) { - next.setDate(next.getDate() + 1); - } - - return next; -} +const scheduler = createScheduler("visa-sponsors", async () => { + await downloadLatestCsv(); +}); /** * Get the next scheduled update time as ISO string */ -function getNextScheduledUpdate(): string | null { - return nextScheduledUpdateTime?.toISOString() || null; -} - -/** - * Schedule the next update - */ -function scheduleNextUpdate(hour = 2): void { - if (scheduledTimer) { - clearTimeout(scheduledTimer); - } - - nextScheduledUpdateTime = calculateNextUpdateTime(hour); - const delay = nextScheduledUpdateTime.getTime() - Date.now(); - - console.log( - `⏰ Next visa sponsor update scheduled for: ${nextScheduledUpdateTime.toISOString()}`, - ); - - scheduledTimer = setTimeout(async () => { - console.log("🔄 Running scheduled visa sponsor update..."); - await downloadLatestCsv(); - scheduleNextUpdate(hour); // Schedule the next one - }, delay); +export function getNextScheduledUpdate(): string | null { + return scheduler.getNextRun(); } /** * Start the scheduler */ export function startScheduler(hour = 2): void { - console.log("🚀 Starting visa sponsor update scheduler..."); - scheduleNextUpdate(hour); + scheduler.start(hour); } /** * Stop the scheduler */ export function stopScheduler(): void { - if (scheduledTimer) { - clearTimeout(scheduledTimer); - scheduledTimer = null; - nextScheduledUpdateTime = null; - console.log("⏹️ Stopped visa sponsor update scheduler"); - } + scheduler.stop(); } /** diff --git a/orchestrator/src/server/utils/scheduler.test.ts b/orchestrator/src/server/utils/scheduler.test.ts new file mode 100644 index 0000000..5f69dc1 --- /dev/null +++ b/orchestrator/src/server/utils/scheduler.test.ts @@ -0,0 +1,195 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { calculateNextTime, createScheduler } from "./scheduler.js"; + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("calculateNextTime", () => { + it("should return today if hour is in the future", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const result = calculateNextTime(14); // 2 PM + + expect(result.getUTCHours()).toBe(14); + expect(result.getUTCDate()).toBe(15); // Same day + expect(result.getTime()).toBeGreaterThan(now.getTime()); + }); + + it("should return tomorrow if hour has passed today", () => { + const now = new Date("2026-01-15T16:00:00Z"); + vi.setSystemTime(now); + + const result = calculateNextTime(10); // 10 AM (already passed) + + expect(result.getUTCHours()).toBe(10); + expect(result.getUTCDate()).toBe(16); // Next day + }); + + it("should return tomorrow if current time equals target hour", () => { + const now = new Date("2026-01-15T14:00:00Z"); + vi.setSystemTime(now); + + const result = calculateNextTime(14); // Same hour + + expect(result.getUTCHours()).toBe(14); + expect(result.getUTCDate()).toBe(16); // Next day (since we're at exactly 14:00) + }); + + it("should handle hour 0 (midnight)", () => { + const now = new Date("2026-01-15T12:00:00Z"); + vi.setSystemTime(now); + + const result = calculateNextTime(0); // Midnight + + expect(result.getUTCHours()).toBe(0); + expect(result.getUTCDate()).toBe(16); // Next day + }); + + it("should handle hour 23", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const result = calculateNextTime(23); + + expect(result.getUTCHours()).toBe(23); + expect(result.getUTCDate()).toBe(15); // Same day + }); +}); + +describe("createScheduler", () => { + it("should create a scheduler with initial stopped state", () => { + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + expect(scheduler.isRunning()).toBe(false); + expect(scheduler.getNextRun()).toBeNull(); + }); + + it("should start scheduling when start() is called", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + scheduler.start(14); + + expect(scheduler.isRunning()).toBe(true); + expect(scheduler.getNextRun()).not.toBeNull(); + }); + + it("should stop scheduling when stop() is called", () => { + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + scheduler.start(14); + expect(scheduler.isRunning()).toBe(true); + + scheduler.stop(); + expect(scheduler.isRunning()).toBe(false); + expect(scheduler.getNextRun()).toBeNull(); + }); + + it("should execute callback after delay (simulated)", async () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + // Start at hour 10 tomorrow (24 hours from now in this test) + scheduler.start(10); + + // Fast-forward time by 24 hours to trigger the callback + await vi.advanceTimersByTimeAsync(24 * 60 * 60 * 1000); + + expect(callback).toHaveBeenCalledTimes(1); + + scheduler.stop(); + }); + + it("should reschedule after execution", async () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + scheduler.start(10); + const firstRun = scheduler.getNextRun(); + + // Fast-forward 24 hours to trigger execution and rescheduling + await vi.advanceTimersByTimeAsync(24 * 60 * 60 * 1000); + + const secondRun = scheduler.getNextRun(); + + // Second run should be 24 hours after first run + expect(secondRun).not.toBe(firstRun); + expect(secondRun).not.toBeNull(); + expect(firstRun).not.toBeNull(); + if (secondRun && firstRun) { + expect(new Date(secondRun).getTime()).toBe( + new Date(firstRun).getTime() + 24 * 60 * 60 * 1000, + ); + } + + scheduler.stop(); + }); + + it("should restart with new hour when start() called again", () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const callback = vi.fn().mockResolvedValue(undefined); + const scheduler = createScheduler("test", callback); + + scheduler.start(14); // 2 PM + const firstRun = scheduler.getNextRun(); + + scheduler.start(16); // 4 PM + const secondRun = scheduler.getNextRun(); + + // Second run should be later than first run + expect(secondRun).not.toBeNull(); + expect(firstRun).not.toBeNull(); + if (secondRun && firstRun) { + expect(new Date(secondRun).getTime()).toBeGreaterThan( + new Date(firstRun).getTime(), + ); + } + + scheduler.stop(); + }); + + it("should handle callback errors gracefully", async () => { + const now = new Date("2026-01-15T10:00:00Z"); + vi.setSystemTime(now); + + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const callback = vi.fn().mockRejectedValue(new Error("Test error")); + const scheduler = createScheduler("test", callback); + + scheduler.start(10); + + // Fast-forward 24 hours to trigger execution + await vi.advanceTimersByTimeAsync(24 * 60 * 60 * 1000); + + expect(callback).toHaveBeenCalled(); + expect(consoleSpy).toHaveBeenCalledWith( + "❌ [test] Scheduled task failed:", + expect.any(Error), + ); + + // Scheduler should still be running after error + expect(scheduler.isRunning()).toBe(true); + + scheduler.stop(); + consoleSpy.mockRestore(); + }); +}); diff --git a/orchestrator/src/server/utils/scheduler.ts b/orchestrator/src/server/utils/scheduler.ts new file mode 100644 index 0000000..fb1c941 --- /dev/null +++ b/orchestrator/src/server/utils/scheduler.ts @@ -0,0 +1,118 @@ +/** + * Shared daily scheduler utility for running tasks at a specific hour. + * Used by visa-sponsors and backup services. + */ + +export interface Scheduler { + /** Start scheduling at the specified hour (0-23) */ + start(hour: number): void; + /** Stop the scheduler */ + stop(): void; + /** Get ISO string of next scheduled run, or null if not running */ + getNextRun(): string | null; + /** Check if scheduler is currently running */ + isRunning(): boolean; +} + +interface SchedulerState { + timer: ReturnType | null; + nextRunTime: Date | null; + currentHour: number | null; +} + +/** + * Calculate the next occurrence of a specific hour (UTC) + * @param hour - Hour of day (0-23) in UTC + * @returns Date object set to the next UTC occurrence of that hour + */ +export function calculateNextTime(hour: number): Date { + const now = new Date(); + const next = new Date(now); + next.setUTCHours(hour, 0, 0, 0); + + // If we've passed the time today, schedule for tomorrow + if (next <= now) { + next.setUTCDate(next.getUTCDate() + 1); + } + + return next; +} + +/** + * Create a reusable daily scheduler + * @param name - Service name for logging + * @param callback - Async function to execute at scheduled time + * @returns Scheduler interface with start/stop/getNextRun methods + */ +export function createScheduler( + name: string, + callback: () => Promise, +): Scheduler { + const state: SchedulerState = { + timer: null, + nextRunTime: null, + currentHour: null, + }; + + function clearState(): void { + if (state.timer) { + clearTimeout(state.timer); + } + state.timer = null; + state.nextRunTime = null; + state.currentHour = null; + } + + function scheduleNext(hour: number): void { + // Clear any existing timer + if (state.timer) { + clearState(); + } + + state.currentHour = hour; + state.nextRunTime = calculateNextTime(hour); + const delay = state.nextRunTime.getTime() - Date.now(); + + console.log( + `⏰ [${name}] Next run scheduled for: ${state.nextRunTime.toISOString()}`, + ); + + state.timer = setTimeout(async () => { + console.log(`🔄 [${name}] Running scheduled task...`); + try { + await callback(); + } catch (error) { + console.error(`❌ [${name}] Scheduled task failed:`, error); + } + // Reschedule for next occurrence + scheduleNext(hour); + }, delay); + } + + return { + start(hour: number): void { + if (state.timer) { + console.log(`🔄 [${name}] Restarting scheduler with hour ${hour}...`); + clearState(); + } else { + console.log(`🚀 [${name}] Starting scheduler at hour ${hour}...`); + } + scheduleNext(hour); + }, + + stop(): void { + if (state.timer) { + clearState(); + console.log(`⏹️ [${name}] Stopped scheduler`); + } + }, + + getNextRun(): string | null { + return state.nextRunTime?.toISOString() || null; + }, + + isRunning(): boolean { + return state.timer !== null; + }, + }; +} diff --git a/orchestrator/src/shared/settings-schema.ts b/orchestrator/src/shared/settings-schema.ts index f752bf5..43fb387 100644 --- a/orchestrator/src/shared/settings-schema.ts +++ b/orchestrator/src/shared/settings-schema.ts @@ -72,6 +72,9 @@ export const updateSettingsSchema = z ukvisajobsPassword: z.string().trim().max(2000).nullable().optional(), webhookSecret: z.string().trim().max(2000).nullable().optional(), enableBasicAuth: z.boolean().optional(), + backupEnabled: z.boolean().nullable().optional(), + backupHour: z.number().int().min(0).max(23).nullable().optional(), + backupMaxCount: z.number().int().min(1).max(5).nullable().optional(), }) .superRefine((data, ctx) => { if (data.enableBasicAuth) { diff --git a/orchestrator/src/shared/types.ts b/orchestrator/src/shared/types.ts index 7dc1e86..8999d37 100644 --- a/orchestrator/src/shared/types.ts +++ b/orchestrator/src/shared/types.ts @@ -545,4 +545,21 @@ export interface AppSettings { ukvisajobsPasswordHint: string | null; webhookSecretHint: string | null; basicAuthActive: boolean; + // Backup settings + backupEnabled: boolean; + defaultBackupEnabled: boolean; + overrideBackupEnabled: boolean | null; + backupHour: number; + defaultBackupHour: number; + overrideBackupHour: number | null; + backupMaxCount: number; + defaultBackupMaxCount: number; + overrideBackupMaxCount: number | null; +} + +export interface BackupInfo { + filename: string; + type: "auto" | "manual"; + size: number; + createdAt: string; }