diff --git a/README.md b/README.md index 1d69635..46a432b 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,43 @@ nanobot agent -m "What is 2+2?" That's it! You have a working AI assistant in 2 minutes. +## 🖥️ Local Models (vLLM) + +Run nanobot with your own local models using vLLM or any OpenAI-compatible server. + +**1. Start your vLLM server** + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000 +``` + +**2. Configure** (`~/.nanobot/config.json`) + +```json +{ + "providers": { + "vllm": { + "apiKey": "dummy", + "apiBase": "http://localhost:8000/v1" + } + }, + "agents": { + "defaults": { + "model": "meta-llama/Llama-3.1-8B-Instruct" + } + } +} +``` + +**3. Chat** + +```bash +nanobot agent -m "Hello from my local LLM!" +``` + +> [!TIP] +> The `apiKey` can be any non-empty string for local servers that don't require authentication. + ## 💬 Chat Apps Talk to your nanobot through Telegram or WhatsApp — anytime, anywhere. diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index 05513f0..1c04d17 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -624,10 +624,13 @@ def status(): has_openrouter = bool(config.providers.openrouter.api_key) has_anthropic = bool(config.providers.anthropic.api_key) has_openai = bool(config.providers.openai.api_key) + has_vllm = bool(config.providers.vllm.api_base) console.print(f"OpenRouter API: {'[green]✓[/green]' if has_openrouter else '[dim]not set[/dim]'}") console.print(f"Anthropic API: {'[green]✓[/green]' if has_anthropic else '[dim]not set[/dim]'}") console.print(f"OpenAI API: {'[green]✓[/green]' if has_openai else '[dim]not set[/dim]'}") + vllm_status = f"[green]✓ {config.providers.vllm.api_base}[/green]" if has_vllm else "[dim]not set[/dim]" + console.print(f"vLLM/Local: {vllm_status}") if __name__ == "__main__": diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index 06c36e6..6414cf9 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -50,6 +50,7 @@ class ProvidersConfig(BaseModel): anthropic: ProviderConfig = Field(default_factory=ProviderConfig) openai: ProviderConfig = Field(default_factory=ProviderConfig) openrouter: ProviderConfig = Field(default_factory=ProviderConfig) + vllm: ProviderConfig = Field(default_factory=ProviderConfig) class GatewayConfig(BaseModel): @@ -88,18 +89,21 @@ class Config(BaseSettings): return Path(self.agents.defaults.workspace).expanduser() def get_api_key(self) -> str | None: - """Get API key in priority order: OpenRouter > Anthropic > OpenAI.""" + """Get API key in priority order: OpenRouter > Anthropic > OpenAI > vLLM.""" return ( self.providers.openrouter.api_key or self.providers.anthropic.api_key or self.providers.openai.api_key or + self.providers.vllm.api_key or None ) def get_api_base(self) -> str | None: - """Get API base URL if using OpenRouter.""" + """Get API base URL if using OpenRouter or vLLM.""" if self.providers.openrouter.api_key: return self.providers.openrouter.api_base or "https://openrouter.ai/api/v1" + if self.providers.vllm.api_base: + return self.providers.vllm.api_base return None class Config: diff --git a/nanobot/providers/litellm_provider.py b/nanobot/providers/litellm_provider.py index c84aa74..4e7305b 100644 --- a/nanobot/providers/litellm_provider.py +++ b/nanobot/providers/litellm_provider.py @@ -32,11 +32,17 @@ class LiteLLMProvider(LLMProvider): (api_base and "openrouter" in api_base) ) + # Track if using custom endpoint (vLLM, etc.) + self.is_vllm = bool(api_base) and not self.is_openrouter + # Configure LiteLLM based on provider if api_key: if self.is_openrouter: # OpenRouter mode - set key os.environ["OPENROUTER_API_KEY"] = api_key + elif self.is_vllm: + # vLLM/custom endpoint - uses OpenAI-compatible API + os.environ["OPENAI_API_KEY"] = api_key elif "anthropic" in default_model: os.environ.setdefault("ANTHROPIC_API_KEY", api_key) elif "openai" in default_model or "gpt" in default_model: @@ -75,6 +81,11 @@ class LiteLLMProvider(LLMProvider): if self.is_openrouter and not model.startswith("openrouter/"): model = f"openrouter/{model}" + # For vLLM, use hosted_vllm/ prefix per LiteLLM docs + # Convert openai/ prefix to hosted_vllm/ if user specified it + if self.is_vllm: + model = f"hosted_vllm/{model}" + kwargs: dict[str, Any] = { "model": model, "messages": messages, @@ -82,6 +93,10 @@ class LiteLLMProvider(LLMProvider): "temperature": temperature, } + # Pass api_base directly for custom endpoints (vLLM, etc.) + if self.api_base: + kwargs["api_base"] = self.api_base + if tools: kwargs["tools"] = tools kwargs["tool_choice"] = "auto"