diff --git a/docs-site/docs/features/settings.md b/docs-site/docs/features/settings.md index 201032b..4026a6d 100644 --- a/docs-site/docs/features/settings.md +++ b/docs-site/docs/features/settings.md @@ -47,6 +47,16 @@ Settings gives you runtime overrides for the key parts of discovery, scoring, ta - Scoring model - Tailoring model - Project-selection model +- Provider defaults are applied automatically when the model fields are left blank: + - `openai` defaults to `gpt-5.4-mini` + - `gemini` defaults to `google/gemini-3-flash-preview` +- The settings page shows provider-aware model pickers for: + - `openai`: available text-generation models only + - `gemini`: available Gemini text-generation models only + - `ollama`: locally installed Ollama models +- `openrouter`, `lmstudio`, and `openai_compatible` stay manual-entry because JobOps cannot safely infer the exact model catalog from those providers +- Changing the provider clears stale model overrides in the form, so inherited fields follow the new provider default unless you explicitly choose a new override +- The preview under each field and the **Resolved config** block reflect the model currently selected in the form, even before you save ### Webhooks @@ -199,6 +209,13 @@ curl -X POST "http://localhost:3001/api/backups" - Some settings apply only to new runs/actions after save. - Re-run scoring/tailoring/pipeline to validate effect. +- In the **Model** section, the field preview and **Resolved config** update immediately when you choose a model, but the change only applies to future actions after you click **Save**. + +### Tailoring or scoring says the selected model does not exist for the current provider + +- Open **Settings -> Model** and confirm the provider and model belong together. +- If you switch providers, leave the model fields blank to use the provider default, or pick a new provider-compatible model from the dropdown. +- JobOps ignores stale Gemini-style overrides under `openai`, and ignores stale OpenAI-style overrides under `gemini`, but you still need to save the current form selection for future runs. ### Resume tailoring used English instead of my resume language diff --git a/docs-site/docs/troubleshooting/common-problems.md b/docs-site/docs/troubleshooting/common-problems.md index 7769d6e..66cdba1 100644 --- a/docs-site/docs/troubleshooting/common-problems.md +++ b/docs-site/docs/troubleshooting/common-problems.md @@ -30,6 +30,14 @@ npm --workspace docs-site run build - Validate `LLM_API_KEY` and provider settings. - Check settings page and API connectivity. +## Resume tailoring or scoring says the model does not exist + +- Root cause: the selected provider and model do not match. +- Open **Settings -> Model** and check both the provider and the current model preview. +- If you recently switched providers, leave the model fields blank to use the provider default, or select a provider-compatible model and save again. +- For `openai`, JobOps defaults to `gpt-5.4-mini` when the model field is blank. +- For `gemini`, JobOps defaults to `google/gemini-3-flash-preview` when the model field is blank. + ## PDF generation fails - Verify RxResume credentials. diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index 6977ee5..37ab50c 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -1336,6 +1336,18 @@ export async function validateLlm(input: { }); } +export async function getLlmModels(input?: { + provider?: string; + baseUrl?: string; + apiKey?: string; +}): Promise { + const data = await fetchApi<{ models: string[] }>("/settings/llm-models", { + method: "POST", + body: JSON.stringify(input ?? {}), + }); + return data.models; +} + export async function validateRxresume(input?: { mode?: "v4" | "v5"; email?: string; diff --git a/orchestrator/src/client/components/OnboardingGate.tsx b/orchestrator/src/client/components/OnboardingGate.tsx index b3dfc66..36810d1 100644 --- a/orchestrator/src/client/components/OnboardingGate.tsx +++ b/orchestrator/src/client/components/OnboardingGate.tsx @@ -17,6 +17,7 @@ import { LLM_PROVIDERS, normalizeLlmProvider, } from "@client/pages/settings/utils"; +import { getDefaultModelForProvider } from "@shared/settings-registry"; import type { UpdateSettingsInput } from "@shared/settings-schema.js"; import type { RxResumeMode, ValidationResult } from "@shared/types.js"; import { Check } from "lucide-react"; @@ -423,6 +424,10 @@ export const OnboardingGate: React.FC = () => { const update: Partial = { llmProvider: normalizedProvider, llmBaseUrl: showBaseUrl ? baseUrlValue || null : null, + model: null, + modelScorer: null, + modelTailoring: null, + modelProjectSelection: null, }; if (showApiKey && apiKeyValue) { @@ -433,7 +438,13 @@ export const OnboardingGate: React.FC = () => { await api.updateSettings(update); await refreshSettings(); setValue("llmApiKey", ""); - toast.success("LLM provider connected"); + const defaultModel = getDefaultModelForProvider(normalizedProvider); + toast.success("LLM provider connected", { + description: + normalizedProvider === "openai" || normalizedProvider === "gemini" + ? `Default for ${providerConfig.label}: ${defaultModel}.` + : "Select the model manually in Settings > Model.", + }); return true; } catch (error) { const message = diff --git a/orchestrator/src/client/pages/SettingsPage.test.tsx b/orchestrator/src/client/pages/SettingsPage.test.tsx index 18b57a4..2454542 100644 --- a/orchestrator/src/client/pages/SettingsPage.test.tsx +++ b/orchestrator/src/client/pages/SettingsPage.test.tsx @@ -15,6 +15,7 @@ const render = (ui: Parameters[0]) => vi.mock("../api", () => ({ getSettings: vi.fn(), + getLlmModels: vi.fn().mockResolvedValue([]), updateSettings: vi.fn(), validateRxresume: vi.fn(), getRxResumeProjects: vi.fn(), @@ -217,6 +218,50 @@ describe("SettingsPage", () => { await waitFor(() => expect(saveButton).toBeEnabled()); }); + it("clears stale model overrides when the provider changes", async () => { + vi.mocked(api.getSettings).mockResolvedValue( + createAppSettings({ + model: { + value: "google/gemini-3-flash-preview", + default: "google/gemini-3-flash-preview", + override: "google/gemini-3-flash-preview", + }, + modelScorer: { value: "google/gemini-3-flash-preview", override: null }, + modelTailoring: { + value: "google/gemini-3-flash-preview", + override: "google/gemini-3-flash-preview", + }, + modelProjectSelection: { + value: "google/gemini-3-flash-preview", + override: null, + }, + llmProvider: { value: "gemini", default: "gemini", override: "gemini" }, + }), + ); + vi.mocked(api.updateSettings).mockResolvedValue(baseSettings); + + renderPage(); + await openModelSection(); + + fireEvent.click(screen.getByRole("combobox", { name: /provider/i })); + fireEvent.click(await screen.findByText("OpenAI")); + + const saveButton = screen.getByRole("button", { name: /^save$/i }); + await waitFor(() => expect(saveButton).toBeEnabled()); + fireEvent.click(saveButton); + + await waitFor(() => expect(api.updateSettings).toHaveBeenCalled()); + expect(api.updateSettings).toHaveBeenCalledWith( + expect.objectContaining({ + llmProvider: "openai", + model: null, + modelScorer: null, + modelTailoring: null, + modelProjectSelection: null, + }), + ); + }); + it("hides pipeline tuning sections that moved to run modal", async () => { vi.mocked(api.getSettings).mockResolvedValue(baseSettings); renderPage(); diff --git a/orchestrator/src/client/pages/SettingsPage.tsx b/orchestrator/src/client/pages/SettingsPage.tsx index b39ef0b..171344f 100644 --- a/orchestrator/src/client/pages/SettingsPage.tsx +++ b/orchestrator/src/client/pages/SettingsPage.tsx @@ -824,10 +824,26 @@ export const SettingsPage: React.FC = () => { } const payload: Partial = { - model: normalizeString(data.model), - modelScorer: normalizeString(data.modelScorer), - modelTailoring: normalizeString(data.modelTailoring), - modelProjectSelection: normalizeString(data.modelProjectSelection), + model: dirtyFields.llmProvider + ? dirtyFields.model + ? normalizeString(data.model) + : null + : normalizeString(data.model), + modelScorer: dirtyFields.llmProvider + ? dirtyFields.modelScorer + ? normalizeString(data.modelScorer) + : null + : normalizeString(data.modelScorer), + modelTailoring: dirtyFields.llmProvider + ? dirtyFields.modelTailoring + ? normalizeString(data.modelTailoring) + : null + : normalizeString(data.modelTailoring), + modelProjectSelection: dirtyFields.llmProvider + ? dirtyFields.modelProjectSelection + ? normalizeString(data.modelProjectSelection) + : null + : normalizeString(data.modelProjectSelection), pipelineWebhookUrl: normalizeString(data.pipelineWebhookUrl), jobCompleteWebhookUrl: normalizeString(data.jobCompleteWebhookUrl), resumeProjects: resumeProjectsOverride, diff --git a/orchestrator/src/client/pages/settings/components/ModelSettingsSection.tsx b/orchestrator/src/client/pages/settings/components/ModelSettingsSection.tsx index d2012e2..eb91760 100644 --- a/orchestrator/src/client/pages/settings/components/ModelSettingsSection.tsx +++ b/orchestrator/src/client/pages/settings/components/ModelSettingsSection.tsx @@ -1,3 +1,4 @@ +import * as api from "@client/api"; import { SettingsInput } from "@client/pages/settings/components/SettingsInput"; import type { ModelValues } from "@client/pages/settings/types"; import { @@ -5,16 +6,19 @@ import { getLlmProviderConfig, LLM_PROVIDER_LABELS, LLM_PROVIDERS, + supportsLlmModelSuggestions, } from "@client/pages/settings/utils"; +import { getDefaultModelForProvider } from "@shared/settings-registry"; import type { UpdateSettingsInput } from "@shared/settings-schema.js"; import type React from "react"; -import { useEffect } from "react"; +import { useDeferredValue, useEffect, useRef, useState } from "react"; import { Controller, useFormContext } from "react-hook-form"; import { AccordionContent, AccordionItem, AccordionTrigger, } from "@/components/ui/accordion"; +import { SearchableDropdown } from "@/components/ui/searchable-dropdown"; import { Select, SelectContent, @@ -35,12 +39,12 @@ export const ModelSettingsSection: React.FC = ({ isLoading, isSaving, }) => { + const [availableModels, setAvailableModels] = useState([]); + const [isLoadingModels, setIsLoadingModels] = useState(false); + const [modelsError, setModelsError] = useState(null); const { effective, default: defaultModel, - scorer, - tailoring, - projectSelection, llmProvider, llmBaseUrl, llmApiKeyHint, @@ -54,10 +58,28 @@ export const ModelSettingsSection: React.FC = ({ } = useFormContext(); const selectedProvider = watch("llmProvider") || llmProvider || "openrouter"; + const previousProviderRef = useRef(selectedProvider); const providerConfig = getLlmProviderConfig(selectedProvider); const { showApiKey, showBaseUrl } = providerConfig; const llmBaseUrlValue = watch("llmBaseUrl"); + const llmApiKeyValue = watch("llmApiKey") ?? ""; + const modelValue = watch("model") ?? ""; + const modelScorerValue = watch("modelScorer") ?? ""; + const modelTailoringValue = watch("modelTailoring") ?? ""; + const modelProjectSelectionValue = watch("modelProjectSelection") ?? ""; + const providerDefaultModel = getDefaultModelForProvider( + selectedProvider, + selectedProvider === llmProvider ? defaultModel : undefined, + ); + const deferredProvider = useDeferredValue(selectedProvider); + const deferredBaseUrl = useDeferredValue(llmBaseUrlValue ?? ""); + const deferredApiKey = useDeferredValue(llmApiKeyValue); + const supportsModelSuggestions = + supportsLlmModelSuggestions(selectedProvider); + const hasAvailableApiKey = showApiKey + ? Boolean(deferredApiKey.trim() || llmApiKeyHint) + : true; useEffect(() => { if (showBaseUrl) return; @@ -66,12 +88,122 @@ export const ModelSettingsSection: React.FC = ({ } }, [setValue, showBaseUrl, llmBaseUrlValue]); + useEffect(() => { + if (previousProviderRef.current === selectedProvider) { + return; + } + + previousProviderRef.current = selectedProvider; + setValue("model", "", { shouldDirty: true }); + setValue("modelScorer", "", { shouldDirty: true }); + setValue("modelTailoring", "", { shouldDirty: true }); + setValue("modelProjectSelection", "", { shouldDirty: true }); + }, [selectedProvider, setValue]); + + useEffect(() => { + if (!supportsModelSuggestions) { + setAvailableModels([]); + setModelsError(null); + setIsLoadingModels(false); + return; + } + + if (!hasAvailableApiKey) { + setAvailableModels([]); + setModelsError(null); + setIsLoadingModels(false); + return; + } + + let cancelled = false; + setIsLoadingModels(true); + setModelsError(null); + + void api + .getLlmModels({ + provider: deferredProvider, + baseUrl: showBaseUrl ? deferredBaseUrl.trim() || undefined : undefined, + apiKey: showApiKey ? deferredApiKey.trim() || undefined : undefined, + }) + .then((models) => { + if (cancelled) return; + setAvailableModels(models); + setModelsError(null); + }) + .catch((error) => { + if (cancelled) return; + setAvailableModels([]); + setModelsError( + error instanceof Error ? error.message : "Failed to load models.", + ); + }) + .finally(() => { + if (cancelled) return; + setIsLoadingModels(false); + }); + + return () => { + cancelled = true; + }; + }, [ + deferredApiKey, + deferredBaseUrl, + deferredProvider, + hasAvailableApiKey, + showApiKey, + showBaseUrl, + supportsModelSuggestions, + ]); + const keyHint = formatSecretHint(llmApiKeyHint); const keyText = showApiKey ? keyHint || "Not set" : "Not required"; - const effectiveDefaultModel = effective || defaultModel || "—"; - const scoringModel = scorer || effectiveDefaultModel; - const tailoringModel = tailoring || effectiveDefaultModel; - const projectSelectionModel = projectSelection || effectiveDefaultModel; + const resolvedBaseUrl = llmBaseUrlValue?.trim() || llmBaseUrl || "-"; + const selectedDefaultModel = modelValue.trim(); + const previewDefaultModel = + selectedDefaultModel || effective || providerDefaultModel || "-"; + const selectedScoringModel = modelScorerValue.trim(); + const selectedTailoringModel = modelTailoringValue.trim(); + const selectedProjectSelectionModel = modelProjectSelectionValue.trim(); + const scoringModel = selectedScoringModel || previewDefaultModel; + const tailoringModel = selectedTailoringModel || previewDefaultModel; + const projectSelectionModel = + selectedProjectSelectionModel || previewDefaultModel; + const modelHelper = supportsModelSuggestions + ? !hasAvailableApiKey + ? `Add or save a ${providerConfig.label} API key to load available models.` + : isLoadingModels + ? "Loading available models..." + : modelsError + ? modelsError + : availableModels.length > 0 + ? "Choose from the available text-generation models." + : "No text-generation models were returned." + : `Type the exact model name manually, or leave blank to use the ${providerConfig.label} default model.`; + const defaultModelOptions = buildModelOptions({ + models: availableModels, + emptyLabel: `Use ${providerConfig.label} default`, + emptyValue: "", + fallbackValue: modelValue.trim(), + }); + const scoringModelOptions = buildModelOptions({ + models: availableModels, + emptyLabel: "Inherit default model", + emptyValue: "", + fallbackValue: modelScorerValue.trim(), + }); + const tailoringModelOptions = buildModelOptions({ + models: availableModels, + emptyLabel: "Inherit default model", + emptyValue: "", + fallbackValue: modelTailoringValue.trim(), + }); + const projectSelectionModelOptions = buildModelOptions({ + models: availableModels, + emptyLabel: "Inherit default model", + emptyValue: "", + fallbackValue: modelProjectSelectionValue.trim(), + }); + return ( @@ -128,7 +260,7 @@ export const ModelSettingsSection: React.FC = ({ disabled={isLoading || isSaving} error={errors.llmBaseUrl?.message as string | undefined} helper={providerConfig.baseUrlHelper} - current={llmBaseUrl || "—"} + current={resolvedBaseUrl} /> )} {showApiKey && ( @@ -147,15 +279,53 @@ export const ModelSettingsSection: React.FC = ({ - + {supportsModelSuggestions ? ( +
+ + ( + + )} + /> + {errors.model?.message && ( +

+ {errors.model.message as string} +

+ )} +
{modelHelper}
+
+ Current:{" "} + {previewDefaultModel} +
+
+ ) : ( + + )} @@ -163,34 +333,161 @@ export const ModelSettingsSection: React.FC = ({
Task-Specific Overrides
- + {supportsModelSuggestions ? ( + <> +
+ + ( + + )} + /> + {errors.modelScorer?.message && ( +

+ {errors.modelScorer.message as string} +

+ )} +
+ Current: {scoringModel} +
+
- +
+ + ( + + )} + /> + {errors.modelTailoring?.message && ( +

+ {errors.modelTailoring.message as string} +

+ )} +
+ Current:{" "} + {tailoringModel} +
+
- +
+ + ( + + )} + /> + {errors.modelProjectSelection?.message && ( +

+ {errors.modelProjectSelection.message as string} +

+ )} +
+ Current:{" "} + {projectSelectionModel} +
+
+ + ) : ( + <> + + + + + + + )}
@@ -200,36 +497,32 @@ export const ModelSettingsSection: React.FC = ({
Resolved config
Provider
-
{selectedProvider || "—"}
+
{selectedProvider || "-"}
Base URL
-
{llmBaseUrl || "—"}
+
{resolvedBaseUrl}
API key
{keyText}
Default model
-
{effectiveDefaultModel}
+
{previewDefaultModel}
Scoring model
- {scoringModel === effectiveDefaultModel - ? "inherits" - : scoringModel} + {selectedScoringModel ? scoringModel : "inherits"}
Tailoring model
- {tailoringModel === effectiveDefaultModel - ? "inherits" - : tailoringModel} + {selectedTailoringModel ? tailoringModel : "inherits"}
Project selection
- {projectSelectionModel === effectiveDefaultModel - ? "inherits" - : projectSelectionModel} + {selectedProjectSelectionModel + ? projectSelectionModel + : "inherits"}
@@ -238,3 +531,37 @@ export const ModelSettingsSection: React.FC = ({ ); }; + +function buildModelOptions(input: { + models: string[]; + emptyLabel: string; + emptyValue: string; + fallbackValue?: string; +}) { + const options = [ + { + value: input.emptyValue, + label: input.emptyLabel, + searchText: input.emptyLabel, + }, + ...input.models.map((model) => ({ + value: model, + label: model, + searchText: model, + })), + ]; + + const fallbackValue = input.fallbackValue?.trim(); + if ( + fallbackValue && + !options.some((option) => option.value === fallbackValue) + ) { + options.unshift({ + value: fallbackValue, + label: fallbackValue, + searchText: `${fallbackValue} custom`, + }); + } + + return options; +} diff --git a/orchestrator/src/client/pages/settings/utils.test.ts b/orchestrator/src/client/pages/settings/utils.test.ts index 97a7e80..a097cb5 100644 --- a/orchestrator/src/client/pages/settings/utils.test.ts +++ b/orchestrator/src/client/pages/settings/utils.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { getLlmProviderConfig, normalizeLlmProvider } from "./utils"; +import { + getLlmProviderConfig, + normalizeLlmProvider, + supportsLlmModelSuggestions, +} from "./utils"; describe("settings utils", () => { it("treats openai-compatible as a dedicated configurable provider", () => { @@ -20,4 +24,11 @@ describe("settings utils", () => { it("defaults unknown providers to openrouter", () => { expect(normalizeLlmProvider("unknown-provider")).toBe("openrouter"); }); + + it("only enables model suggestions for supported providers", () => { + expect(supportsLlmModelSuggestions("openai")).toBe(true); + expect(supportsLlmModelSuggestions("gemini")).toBe(true); + expect(supportsLlmModelSuggestions("ollama")).toBe(true); + expect(supportsLlmModelSuggestions("openrouter")).toBe(false); + }); }); diff --git a/orchestrator/src/client/pages/settings/utils.ts b/orchestrator/src/client/pages/settings/utils.ts index 4ef9895..92632fa 100644 --- a/orchestrator/src/client/pages/settings/utils.ts +++ b/orchestrator/src/client/pages/settings/utils.ts @@ -29,6 +29,11 @@ export const LLM_PROVIDERS = [ ] as const; export type LlmProviderId = (typeof LLM_PROVIDERS)[number]; +export const LLM_MODEL_SUGGESTION_PROVIDERS = [ + "openai", + "gemini", + "ollama", +] as const; export const LLM_PROVIDER_LABELS: Record = { openrouter: "OpenRouter", @@ -92,6 +97,15 @@ export function normalizeLlmProvider( : "openrouter"; } +export function supportsLlmModelSuggestions( + provider: string | null | undefined, +): boolean { + const normalizedProvider = normalizeLlmProvider(provider); + return (LLM_MODEL_SUGGESTION_PROVIDERS as readonly string[]).includes( + normalizedProvider, + ); +} + export function getLlmProviderConfig(provider: string | null | undefined) { const normalizedProvider = normalizeLlmProvider(provider); const showApiKey = PROVIDERS_WITH_API_KEY.has(normalizedProvider); diff --git a/orchestrator/src/components/ui/searchable-dropdown.tsx b/orchestrator/src/components/ui/searchable-dropdown.tsx index b05d347..0dabbe3 100644 --- a/orchestrator/src/components/ui/searchable-dropdown.tsx +++ b/orchestrator/src/components/ui/searchable-dropdown.tsx @@ -24,6 +24,7 @@ export interface SearchableDropdownOption { } interface SearchableDropdownProps { + inputId?: string; value: string; options: SearchableDropdownOption[]; onValueChange: (value: string) => void; @@ -38,6 +39,7 @@ interface SearchableDropdownProps { } export const SearchableDropdown: React.FC = ({ + inputId, value, options, onValueChange, @@ -51,18 +53,46 @@ export const SearchableDropdown: React.FC = ({ listClassName, }) => { const [open, setOpen] = React.useState(false); + const [query, setQuery] = React.useState(""); const selectedOption = options.find((option) => option.value === value); - const triggerLabel = selectedOption?.label ?? placeholder; + const trimmedQuery = query.trim(); + const hasCustomValue = + trimmedQuery.length > 0 && + !options.some( + (option) => + option.value === trimmedQuery || option.label.trim() === trimmedQuery, + ); + const triggerLabel = selectedOption?.label ?? (value || placeholder); return ( - + { + setOpen(nextOpen); + if (!nextOpen) { + setQuery(""); + } + }} + > + {inputId ? ( + onValueChange(event.target.value)} + className="sr-only" + tabIndex={-1} + aria-hidden="true" + /> + ) : null}