Use project venv in CI to satisfy PEP 668 on Ubuntu runner.

Gitea act_runner images block system-wide pip; install deps into .venv and invoke tools from .venv/bin.

.dockerignore

@@ -45,7 +45,6 @@ logs/
 .DS_Store
 Thumbs.db
 
-# Docs (optional - include if you want them in container)
+# Docs (keep README.md — required by pyproject.toml / Docker build)
 docs/
-*.md
 

.github/workflows/ci.yml

@@ -9,8 +9,7 @@ on:
 jobs:
   lint-and-test:
     runs-on: ubuntu-latest
-    container:
-      image: python:3.11-bullseye
+    # No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images)
 
     services:
       postgres:
@@ -29,24 +28,20 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install system dependencies
+      - name: Set up Python venv
         run: |
-          apt-get update
-          apt-get install -y postgresql-client
-
-      - name: Install Python dependencies
-        run: |
-          pip install --upgrade pip
-          pip install -e ".[dev]"
+          python3 -m venv .venv
+          .venv/bin/pip install --upgrade pip
+          .venv/bin/pip install -e ".[dev]"
 
       - name: Run linters
         run: |
           echo "Running ruff..."
-          ruff check src/ tests/ || true
+          .venv/bin/ruff check src/ tests/ || true
           echo "Running black check..."
-          black --check src/ tests/ || true
+          .venv/bin/black --check src/ tests/ || true
           echo "Running mypy..."
-          mypy src/ --install-types --non-interactive || true
+          .venv/bin/mypy src/ --install-types --non-interactive || true
 
       - name: Run tests with coverage
         env:
@@ -57,40 +52,38 @@ jobs:
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }}
           FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }}
         run: |
-          pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
+          .venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
 
       - name: Test scripts
         env:
           DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test
         run: |
           echo "Testing database migrations..."
-          alembic upgrade head
+          .venv/bin/alembic upgrade head
           echo "Testing price loader..."
-          python scripts/fetch_sample_prices.py || true
+          .venv/bin/python scripts/fetch_sample_prices.py || true
 
   security-scan:
     runs-on: ubuntu-latest
-    container:
-      image: python:3.11-bullseye
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install dependencies
+      - name: Set up Python venv
         run: |
-          pip install --upgrade pip
-          pip install safety bandit
+          python3 -m venv .venv
+          .venv/bin/pip install --upgrade pip
+          .venv/bin/pip install -e ".[dev]" safety bandit
 
       - name: Run safety check
         run: |
-          pip install -e .
-          safety check --json || true
+          .venv/bin/safety check --json || true
         continue-on-error: true
 
       - name: Run bandit security scan
         run: |
-          bandit -r src/ -f json -o bandit-report.json || true
-          bandit -r src/ -f screen
+          .venv/bin/bandit -r src/ -f json -o bandit-report.json || true
+          .venv/bin/bandit -r src/ -f screen
         continue-on-error: true
 
   dependency-scan:
@@ -114,20 +107,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: false
-          tags: pote:test
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Test Docker image
+      - name: Build and test Docker image
         run: |
+          # Plain docker build — buildx images are not visible to act_runner's docker run
+          docker build -t pote:test .
           docker run --rm pote:test python -c "import pote; print('POTE import successful')"
 
   workflow-summary:

EMAIL_SETUP.md

@@ -1,18 +1,18 @@
-# Email Setup for levkin.ca
+# Email Setup for levkine.ca (Mailcow)
 
-Your POTE system is configured to use `test@levkin.ca` for sending reports.
+Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`.
 
 ## ✅ Configuration Done
 
 The `.env` file has been created with these settings:
 
 ```env
-SMTP_HOST=mail.levkin.ca
+SMTP_HOST=10.0.10.132
 SMTP_PORT=587
-SMTP_USER=test@levkin.ca
+SMTP_USER=alerts@levkine.ca
 SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE
-FROM_EMAIL=test@levkin.ca
-REPORT_RECIPIENTS=test@levkin.ca
+FROM_EMAIL=alerts@levkine.ca
+REPORT_RECIPIENTS=idobkin@gmail.com
 ```
 
 ## 🔑 Next Steps

pyproject.toml

@@ -18,8 +18,10 @@ dependencies = [
     "pydantic-settings>=2.0",
     "python-dotenv>=1.0",
     "requests>=2.31",
+    "httpx>=0.27",
     "pandas>=2.0",
     "numpy>=1.24",
+    "scikit-learn>=1.3",
     "yfinance>=0.2",
     "psycopg2-binary>=2.9",
 ]

scripts/fetch_congressional_trades.py

@@ -28,8 +28,8 @@ def main():
 
     args = parser.parse_args()
 
-    logger.info("=== Fetching Congressional Trades from House Stock Watcher ===")
-    logger.info("Source: https://housestockwatcher.com (free, no API key)")
+    logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===")
+    logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)")
 
     try:
         with HouseWatcherClient() as client:

src/pote/config.py

@@ -30,6 +30,16 @@ class Settings(BaseSettings):
     # Logging
     log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)")
 
+    # Email (Mailcow @ mail.levkine.ca — use LAN IP from app LXCs if DNS points public)
+    smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname")
+    smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)")
+    smtp_user: str = Field(default="", description="SMTP auth username")
+    smtp_password: str = Field(default="", description="SMTP auth password")
+    from_email: str = Field(default="", description="From address for reports")
+    report_recipients: str = Field(
+        default="", description="Default report recipients (comma-separated)"
+    )
+
     # Application
     app_name: str = "POTE"
     app_version: str = "0.1.0"

src/pote/db/__init__.py

@@ -3,6 +3,7 @@ Database layer: engine, session factory, and base model.
 """
 
 from collections.abc import Generator
+from contextlib import contextmanager
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
@@ -26,8 +27,9 @@ class Base(DeclarativeBase):
     pass
 
 
+@contextmanager
 def get_session() -> Generator[Session, None, None]:
-    """Get a database session (use as a context manager or dependency)."""
+    """Get a database session (context manager or FastAPI-style dependency)."""
     session = SessionLocal()
     try:
         yield session

src/pote/ingestion/house_watcher.py

@@ -1,9 +1,12 @@
 """
 House Stock Watcher client for fetching congressional trade data.
-Free, no API key required - scrapes from housestockwatcher.com
+
+Uses public STOCK Act disclosure datasets (no API key). The legacy
+housestockwatcher.com host is often unavailable; we try several mirrors.
 """
 
 import logging
+import os
 from datetime import date, datetime
 from typing import Any
 
@@ -11,16 +14,69 @@ import httpx
 
 logger = logging.getLogger(__name__)
 
+def _default_data_urls() -> tuple[str, ...]:
+    override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip()
+    if override:
+        return (override,)
+    return (
+        "https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json",
+        "https://housestockwatcher.com/api/all_transactions",
+        "https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json",
+    )
+
+
+DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls()
+
+
+def _ascii_safe(text: str | None) -> str:
+    """Normalize text for DB fields that may use ASCII-only client encoding."""
+    if not text:
+        return ""
+    return text.replace("\u00b7", "-").replace("·", "-").strip()
+
+
+def _party_label(code: str | None) -> str:
+    if not code:
+        return ""
+    mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"}
+    return mapping.get(code.strip().upper(), code.strip())
+
+
+def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]:
+    """
+    Map a raw disclosure record to the House Stock Watcher field names
+    expected by TradeLoader.
+    """
+    if "representative" in raw and "disclosure_date" in raw:
+        return raw
+
+    # Kadoa congress-trading-monitor export
+    if "filer_name" in raw:
+        chamber = (raw.get("chamber") or "").strip().lower()
+        house = "Senate" if chamber == "senate" else "House"
+        return {
+            "representative": raw.get("filer_name", "").strip(),
+            "ticker": (raw.get("ticker") or "").strip(),
+            "transaction_date": raw.get("transaction_date", ""),
+            "disclosure_date": raw.get("filing_date", ""),
+            "transaction": raw.get("transaction_type", ""),
+            "amount": raw.get("amount_range_label", ""),
+            "house": house,
+            "district": _ascii_safe(raw.get("office")),
+            "party": _party_label(raw.get("party")),
+        }
+
+    return raw
+
 
 class HouseWatcherClient:
     """
-    Client for House Stock Watcher API (free, community-maintained).
+    Client for congressional trade JSON feeds (free, community-maintained).
 
-    Data source: https://housestockwatcher.com/
-    No authentication required.
+    Primary source: congress-trading-monitor public dataset on GitHub.
     """
 
-    BASE_URL = "https://housestockwatcher.com/api"
+    data_urls: tuple[str, ...] = DEFAULT_DATA_URLS
 
     def __init__(self, timeout: float = 30.0):
         """
@@ -65,30 +121,31 @@ class HouseWatcherClient:
         Raises:
             httpx.HTTPError: If request fails
         """
-        url = f"{self.BASE_URL}/all_transactions"
+        last_error: Exception | None = None
+        for url in self.data_urls:
+            if not url:
+                continue
             logger.info(f"Fetching transactions from {url}")
-
             try:
                 response = self._client.get(url)
                 response.raise_for_status()
                 data = response.json()
-
                 if not isinstance(data, list):
                     raise ValueError(f"Expected list response, got {type(data)}")
-
-            logger.info(f"Fetched {len(data)} transactions from House Stock Watcher")
-
+                data = [normalize_transaction_record(item) for item in data]
+                logger.info(f"Fetched {len(data)} transactions from {url}")
                 if limit:
                     data = data[:limit]
-
                 return data
-
-        except httpx.HTTPError as e:
-            logger.error(f"Failed to fetch from House Stock Watcher: {e}")
-            raise
             except Exception as e:
-            logger.error(f"Unexpected error fetching transactions: {e}")
-            raise
+                last_error = e
+                logger.warning(f"Failed to fetch from {url}: {e}")
+                continue
+
+        logger.error("Failed to fetch congressional trades from all configured URLs")
+        if last_error:
+            raise last_error
+        raise RuntimeError("No data URLs configured")
 
     def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]:
         """

src/pote/ingestion/trade_loader.py

@@ -45,12 +45,11 @@ class TradeLoader:
 
         for txn in transactions:
             try:
-                # Get or create official
+                with self.session.begin_nested():
                     official, is_new_official = self._get_or_create_official(txn)
                     if is_new_official:
                         officials_created += 1
 
-                # Get or create security
                     ticker = txn.get("ticker", "").strip().upper()
                     if not ticker or ticker in ("N/A", "--", ""):
                         logger.debug(
@@ -62,7 +61,6 @@ class TradeLoader:
                     if is_new_security:
                         securities_created += 1
 
-                # Create trade (upsert)
                     trade_created = self._upsert_trade(txn, official.id, security.id, source)
                     if trade_created:
                         trades_created += 1