From 2b19dcf9fd4281115a3d55034e1e1fdd3ec74d66 Mon Sep 17 00:00:00 2001 From: ZhihaoZhang97 Date: Mon, 2 Feb 2026 11:23:04 +1100 Subject: [PATCH 1/3] feat: add vLLM/local LLM support - Add vllm provider configuration in config schema - Auto-detect vLLM endpoints and use hosted_vllm/ prefix for LiteLLM - Pass api_base directly to acompletion for custom endpoints - Add vLLM status display in CLI status command - Add vLLM setup documentation in README --- README.md | 37 +++++++++++++++++++++++++++ nanobot/cli/commands.py | 3 +++ nanobot/config/schema.py | 8 ++++-- nanobot/providers/litellm_provider.py | 15 +++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d69635..46a432b 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,43 @@ nanobot agent -m "What is 2+2?" That's it! You have a working AI assistant in 2 minutes. +## 🖥️ Local Models (vLLM) + +Run nanobot with your own local models using vLLM or any OpenAI-compatible server. + +**1. Start your vLLM server** + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000 +``` + +**2. Configure** (`~/.nanobot/config.json`) + +```json +{ + "providers": { + "vllm": { + "apiKey": "dummy", + "apiBase": "http://localhost:8000/v1" + } + }, + "agents": { + "defaults": { + "model": "meta-llama/Llama-3.1-8B-Instruct" + } + } +} +``` + +**3. Chat** + +```bash +nanobot agent -m "Hello from my local LLM!" +``` + +> [!TIP] +> The `apiKey` can be any non-empty string for local servers that don't require authentication. + ## 💬 Chat Apps Talk to your nanobot through Telegram or WhatsApp — anytime, anywhere. diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index 05513f0..1c04d17 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -624,10 +624,13 @@ def status(): has_openrouter = bool(config.providers.openrouter.api_key) has_anthropic = bool(config.providers.anthropic.api_key) has_openai = bool(config.providers.openai.api_key) + has_vllm = bool(config.providers.vllm.api_base) console.print(f"OpenRouter API: {'[green]✓[/green]' if has_openrouter else '[dim]not set[/dim]'}") console.print(f"Anthropic API: {'[green]✓[/green]' if has_anthropic else '[dim]not set[/dim]'}") console.print(f"OpenAI API: {'[green]✓[/green]' if has_openai else '[dim]not set[/dim]'}") + vllm_status = f"[green]✓ {config.providers.vllm.api_base}[/green]" if has_vllm else "[dim]not set[/dim]" + console.print(f"vLLM/Local: {vllm_status}") if __name__ == "__main__": diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index 06c36e6..6414cf9 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -50,6 +50,7 @@ class ProvidersConfig(BaseModel): anthropic: ProviderConfig = Field(default_factory=ProviderConfig) openai: ProviderConfig = Field(default_factory=ProviderConfig) openrouter: ProviderConfig = Field(default_factory=ProviderConfig) + vllm: ProviderConfig = Field(default_factory=ProviderConfig) class GatewayConfig(BaseModel): @@ -88,18 +89,21 @@ class Config(BaseSettings): return Path(self.agents.defaults.workspace).expanduser() def get_api_key(self) -> str | None: - """Get API key in priority order: OpenRouter > Anthropic > OpenAI.""" + """Get API key in priority order: OpenRouter > Anthropic > OpenAI > vLLM.""" return ( self.providers.openrouter.api_key or self.providers.anthropic.api_key or self.providers.openai.api_key or + self.providers.vllm.api_key or None ) def get_api_base(self) -> str | None: - """Get API base URL if using OpenRouter.""" + """Get API base URL if using OpenRouter or vLLM.""" if self.providers.openrouter.api_key: return self.providers.openrouter.api_base or "https://openrouter.ai/api/v1" + if self.providers.vllm.api_base: + return self.providers.vllm.api_base return None class Config: diff --git a/nanobot/providers/litellm_provider.py b/nanobot/providers/litellm_provider.py index c84aa74..4e7305b 100644 --- a/nanobot/providers/litellm_provider.py +++ b/nanobot/providers/litellm_provider.py @@ -32,11 +32,17 @@ class LiteLLMProvider(LLMProvider): (api_base and "openrouter" in api_base) ) + # Track if using custom endpoint (vLLM, etc.) + self.is_vllm = bool(api_base) and not self.is_openrouter + # Configure LiteLLM based on provider if api_key: if self.is_openrouter: # OpenRouter mode - set key os.environ["OPENROUTER_API_KEY"] = api_key + elif self.is_vllm: + # vLLM/custom endpoint - uses OpenAI-compatible API + os.environ["OPENAI_API_KEY"] = api_key elif "anthropic" in default_model: os.environ.setdefault("ANTHROPIC_API_KEY", api_key) elif "openai" in default_model or "gpt" in default_model: @@ -75,6 +81,11 @@ class LiteLLMProvider(LLMProvider): if self.is_openrouter and not model.startswith("openrouter/"): model = f"openrouter/{model}" + # For vLLM, use hosted_vllm/ prefix per LiteLLM docs + # Convert openai/ prefix to hosted_vllm/ if user specified it + if self.is_vllm: + model = f"hosted_vllm/{model}" + kwargs: dict[str, Any] = { "model": model, "messages": messages, @@ -82,6 +93,10 @@ class LiteLLMProvider(LLMProvider): "temperature": temperature, } + # Pass api_base directly for custom endpoints (vLLM, etc.) + if self.api_base: + kwargs["api_base"] = self.api_base + if tools: kwargs["tools"] = tools kwargs["tool_choice"] = "auto" From 4ba8cc0f8f8b5adcbc75bddba5082596e3a5ea50 Mon Sep 17 00:00:00 2001 From: chaohuang-ai Date: Mon, 2 Feb 2026 12:59:36 +0800 Subject: [PATCH 2/3] Update README.md --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 46a432b..167ae22 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,24 @@ nanobot/ **Want to help?** Pick an item and [open a PR](https://github.com/HKUDS/nanobot/pulls)! +--- + +## ⭐ Star History + +*Community Growth Trajectory* + +
+ + + + + Star History Chart + + +
+ +--- + ## 🤝 Contribute PRs welcome! The codebase is intentionally small and readable. 🤗 From c400786b17bbab67d2d4a01fede13191d274699e Mon Sep 17 00:00:00 2001 From: Neutral Milk Date: Mon, 2 Feb 2026 13:35:44 +0800 Subject: [PATCH 3/3] chore: change default gateway port to 18790 to avoid conflict with OpenClaw --- nanobot/cli/commands.py | 2 +- nanobot/config/schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index 05513f0..f8d3b98 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -154,7 +154,7 @@ This file stores important information that should persist across sessions. @app.command() def gateway( - port: int = typer.Option(18789, "--port", "-p", help="Gateway port"), + port: int = typer.Option(18790, "--port", "-p", help="Gateway port"), verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose output"), ): """Start the nanobot gateway.""" diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index 06c36e6..2931fd5 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -55,7 +55,7 @@ class ProvidersConfig(BaseModel): class GatewayConfig(BaseModel): """Gateway/server configuration.""" host: str = "0.0.0.0" - port: int = 18789 + port: int = 18790 class WebSearchConfig(BaseModel):