Merge pull request #4 from ZhihaoZhang97/feature/vllm-support

feat: add vLLM/local LLM support

README.md

@@ -116,6 +116,43 @@ nanobot agent -m "What is 2+2?"
 
 That's it! You have a working AI assistant in 2 minutes.
 
+## 🖥️ Local Models (vLLM)
+
+Run nanobot with your own local models using vLLM or any OpenAI-compatible server.
+
+**1. Start your vLLM server**
+
+```bash
+vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000
+```
+
+**2. Configure** (`~/.nanobot/config.json`)
+
+```json
+{
+  "providers": {
+    "vllm": {
+      "apiKey": "dummy",
+      "apiBase": "http://localhost:8000/v1"
+    }
+  },
+  "agents": {
+    "defaults": {
+      "model": "meta-llama/Llama-3.1-8B-Instruct"
+    }
+  }
+}
+```
+
+**3. Chat**
+
+```bash
+nanobot agent -m "Hello from my local LLM!"
+```
+
+> [!TIP]
+> The `apiKey` can be any non-empty string for local servers that don't require authentication.
+
 ## 💬 Chat Apps
 
 Talk to your nanobot through Telegram or WhatsApp — anytime, anywhere.

nanobot/cli/commands.py

@@ -624,10 +624,13 @@ def status():
         has_openrouter = bool(config.providers.openrouter.api_key)
         has_anthropic = bool(config.providers.anthropic.api_key)
         has_openai = bool(config.providers.openai.api_key)
+        has_vllm = bool(config.providers.vllm.api_base)
         
         console.print(f"OpenRouter API: {'[green]✓[/green]' if has_openrouter else '[dim]not set[/dim]'}")
         console.print(f"Anthropic API: {'[green]✓[/green]' if has_anthropic else '[dim]not set[/dim]'}")
         console.print(f"OpenAI API: {'[green]✓[/green]' if has_openai else '[dim]not set[/dim]'}")
+        vllm_status = f"[green]✓ {config.providers.vllm.api_base}[/green]" if has_vllm else "[dim]not set[/dim]"
+        console.print(f"vLLM/Local: {vllm_status}")
 
 
 if __name__ == "__main__":

nanobot/config/schema.py

@@ -50,6 +50,7 @@ class ProvidersConfig(BaseModel):
     anthropic: ProviderConfig = Field(default_factory=ProviderConfig)
     openai: ProviderConfig = Field(default_factory=ProviderConfig)
     openrouter: ProviderConfig = Field(default_factory=ProviderConfig)
+    vllm: ProviderConfig = Field(default_factory=ProviderConfig)
 
 
 class GatewayConfig(BaseModel):
@@ -88,18 +89,21 @@ class Config(BaseSettings):
         return Path(self.agents.defaults.workspace).expanduser()
     
     def get_api_key(self) -> str | None:
-        """Get API key in priority order: OpenRouter > Anthropic > OpenAI."""
+        """Get API key in priority order: OpenRouter > Anthropic > OpenAI > vLLM."""
         return (
             self.providers.openrouter.api_key or
             self.providers.anthropic.api_key or
             self.providers.openai.api_key or
+            self.providers.vllm.api_key or
             None
         )
     
     def get_api_base(self) -> str | None:
-        """Get API base URL if using OpenRouter."""
+        """Get API base URL if using OpenRouter or vLLM."""
         if self.providers.openrouter.api_key:
             return self.providers.openrouter.api_base or "https://openrouter.ai/api/v1"
+        if self.providers.vllm.api_base:
+            return self.providers.vllm.api_base
         return None
     
     class Config:

nanobot/providers/litellm_provider.py

@@ -32,11 +32,17 @@ class LiteLLMProvider(LLMProvider):
             (api_base and "openrouter" in api_base)
         )
         
+        # Track if using custom endpoint (vLLM, etc.)
+        self.is_vllm = bool(api_base) and not self.is_openrouter
+        
         # Configure LiteLLM based on provider
         if api_key:
             if self.is_openrouter:
                 # OpenRouter mode - set key
                 os.environ["OPENROUTER_API_KEY"] = api_key
+            elif self.is_vllm:
+                # vLLM/custom endpoint - uses OpenAI-compatible API
+                os.environ["OPENAI_API_KEY"] = api_key
             elif "anthropic" in default_model:
                 os.environ.setdefault("ANTHROPIC_API_KEY", api_key)
             elif "openai" in default_model or "gpt" in default_model:
@@ -75,6 +81,11 @@ class LiteLLMProvider(LLMProvider):
         if self.is_openrouter and not model.startswith("openrouter/"):
             model = f"openrouter/{model}"
         
+        # For vLLM, use hosted_vllm/ prefix per LiteLLM docs
+        # Convert openai/ prefix to hosted_vllm/ if user specified it
+        if self.is_vllm:
+            model = f"hosted_vllm/{model}"
+        
         kwargs: dict[str, Any] = {
             "model": model,
             "messages": messages,
@@ -82,6 +93,10 @@ class LiteLLMProvider(LLMProvider):
             "temperature": temperature,
         }
         
+        # Pass api_base directly for custom endpoints (vLLM, etc.)
+        if self.api_base:
+            kwargs["api_base"] = self.api_base
+        
         if tools:
             kwargs["tools"] = tools
             kwargs["tool_choice"] = "auto"