From 31c656d66f3086290099ff1c8a20eca2c751562a Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 26 May 2026 19:52:29 -0400 Subject: [PATCH 1/4] Fix live ingest, email config, and dependencies for homelab deploy. Switch HouseWatcherClient to public STOCK Act JSON (legacy housestockwatcher.com is down), add httpx/scikit-learn to pyproject, wire SMTP settings through Settings, fix get_session() as a context manager, and use savepoints in TradeLoader so one bad row does not roll back the batch. --- EMAIL_SETUP.md | 12 +-- pyproject.toml | 2 + scripts/fetch_congressional_trades.py | 4 +- src/pote/config.py | 10 +++ src/pote/db/__init__.py | 4 +- src/pote/ingestion/house_watcher.py | 113 +++++++++++++++++++------- src/pote/ingestion/trade_loader.py | 34 ++++---- 7 files changed, 124 insertions(+), 55 deletions(-) diff --git a/EMAIL_SETUP.md b/EMAIL_SETUP.md index 653d9fc..d917e94 100644 --- a/EMAIL_SETUP.md +++ b/EMAIL_SETUP.md @@ -1,18 +1,18 @@ -# Email Setup for levkin.ca +# Email Setup for levkine.ca (Mailcow) -Your POTE system is configured to use `test@levkin.ca` for sending reports. +Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`. ## โœ… Configuration Done The `.env` file has been created with these settings: ```env -SMTP_HOST=mail.levkin.ca +SMTP_HOST=10.0.10.132 SMTP_PORT=587 -SMTP_USER=test@levkin.ca +SMTP_USER=alerts@levkine.ca SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE -FROM_EMAIL=test@levkin.ca -REPORT_RECIPIENTS=test@levkin.ca +FROM_EMAIL=alerts@levkine.ca +REPORT_RECIPIENTS=idobkin@gmail.com ``` ## ๐Ÿ”‘ Next Steps diff --git a/pyproject.toml b/pyproject.toml index 44473ff..8259445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,10 @@ dependencies = [ "pydantic-settings>=2.0", "python-dotenv>=1.0", "requests>=2.31", + "httpx>=0.27", "pandas>=2.0", "numpy>=1.24", + "scikit-learn>=1.3", "yfinance>=0.2", "psycopg2-binary>=2.9", ] diff --git a/scripts/fetch_congressional_trades.py b/scripts/fetch_congressional_trades.py index da8fcfd..33bb4e6 100755 --- a/scripts/fetch_congressional_trades.py +++ b/scripts/fetch_congressional_trades.py @@ -28,8 +28,8 @@ def main(): args = parser.parse_args() - logger.info("=== Fetching Congressional Trades from House Stock Watcher ===") - logger.info("Source: https://housestockwatcher.com (free, no API key)") + logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===") + logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)") try: with HouseWatcherClient() as client: diff --git a/src/pote/config.py b/src/pote/config.py index b35e05c..5abce11 100644 --- a/src/pote/config.py +++ b/src/pote/config.py @@ -30,6 +30,16 @@ class Settings(BaseSettings): # Logging log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)") + # Email (Mailcow @ mail.levkine.ca โ€” use LAN IP from app LXCs if DNS points public) + smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname") + smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)") + smtp_user: str = Field(default="", description="SMTP auth username") + smtp_password: str = Field(default="", description="SMTP auth password") + from_email: str = Field(default="", description="From address for reports") + report_recipients: str = Field( + default="", description="Default report recipients (comma-separated)" + ) + # Application app_name: str = "POTE" app_version: str = "0.1.0" diff --git a/src/pote/db/__init__.py b/src/pote/db/__init__.py index aa3e296..b547fbb 100644 --- a/src/pote/db/__init__.py +++ b/src/pote/db/__init__.py @@ -3,6 +3,7 @@ Database layer: engine, session factory, and base model. """ from collections.abc import Generator +from contextlib import contextmanager from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker @@ -26,8 +27,9 @@ class Base(DeclarativeBase): pass +@contextmanager def get_session() -> Generator[Session, None, None]: - """Get a database session (use as a context manager or dependency).""" + """Get a database session (context manager or FastAPI-style dependency).""" session = SessionLocal() try: yield session diff --git a/src/pote/ingestion/house_watcher.py b/src/pote/ingestion/house_watcher.py index 7892532..ee5e745 100644 --- a/src/pote/ingestion/house_watcher.py +++ b/src/pote/ingestion/house_watcher.py @@ -1,9 +1,12 @@ """ House Stock Watcher client for fetching congressional trade data. -Free, no API key required - scrapes from housestockwatcher.com + +Uses public STOCK Act disclosure datasets (no API key). The legacy +housestockwatcher.com host is often unavailable; we try several mirrors. """ import logging +import os from datetime import date, datetime from typing import Any @@ -11,16 +14,69 @@ import httpx logger = logging.getLogger(__name__) +def _default_data_urls() -> tuple[str, ...]: + override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip() + if override: + return (override,) + return ( + "https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json", + "https://housestockwatcher.com/api/all_transactions", + "https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json", + ) + + +DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls() + + +def _ascii_safe(text: str | None) -> str: + """Normalize text for DB fields that may use ASCII-only client encoding.""" + if not text: + return "" + return text.replace("\u00b7", "-").replace("ยท", "-").strip() + + +def _party_label(code: str | None) -> str: + if not code: + return "" + mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"} + return mapping.get(code.strip().upper(), code.strip()) + + +def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]: + """ + Map a raw disclosure record to the House Stock Watcher field names + expected by TradeLoader. + """ + if "representative" in raw and "disclosure_date" in raw: + return raw + + # Kadoa congress-trading-monitor export + if "filer_name" in raw: + chamber = (raw.get("chamber") or "").strip().lower() + house = "Senate" if chamber == "senate" else "House" + return { + "representative": raw.get("filer_name", "").strip(), + "ticker": (raw.get("ticker") or "").strip(), + "transaction_date": raw.get("transaction_date", ""), + "disclosure_date": raw.get("filing_date", ""), + "transaction": raw.get("transaction_type", ""), + "amount": raw.get("amount_range_label", ""), + "house": house, + "district": _ascii_safe(raw.get("office")), + "party": _party_label(raw.get("party")), + } + + return raw + class HouseWatcherClient: """ - Client for House Stock Watcher API (free, community-maintained). + Client for congressional trade JSON feeds (free, community-maintained). - Data source: https://housestockwatcher.com/ - No authentication required. + Primary source: congress-trading-monitor public dataset on GitHub. """ - BASE_URL = "https://housestockwatcher.com/api" + data_urls: tuple[str, ...] = DEFAULT_DATA_URLS def __init__(self, timeout: float = 30.0): """ @@ -65,30 +121,31 @@ class HouseWatcherClient: Raises: httpx.HTTPError: If request fails """ - url = f"{self.BASE_URL}/all_transactions" - logger.info(f"Fetching transactions from {url}") + last_error: Exception | None = None + for url in self.data_urls: + if not url: + continue + logger.info(f"Fetching transactions from {url}") + try: + response = self._client.get(url) + response.raise_for_status() + data = response.json() + if not isinstance(data, list): + raise ValueError(f"Expected list response, got {type(data)}") + data = [normalize_transaction_record(item) for item in data] + logger.info(f"Fetched {len(data)} transactions from {url}") + if limit: + data = data[:limit] + return data + except Exception as e: + last_error = e + logger.warning(f"Failed to fetch from {url}: {e}") + continue - try: - response = self._client.get(url) - response.raise_for_status() - data = response.json() - - if not isinstance(data, list): - raise ValueError(f"Expected list response, got {type(data)}") - - logger.info(f"Fetched {len(data)} transactions from House Stock Watcher") - - if limit: - data = data[:limit] - - return data - - except httpx.HTTPError as e: - logger.error(f"Failed to fetch from House Stock Watcher: {e}") - raise - except Exception as e: - logger.error(f"Unexpected error fetching transactions: {e}") - raise + logger.error("Failed to fetch congressional trades from all configured URLs") + if last_error: + raise last_error + raise RuntimeError("No data URLs configured") def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]: """ diff --git a/src/pote/ingestion/trade_loader.py b/src/pote/ingestion/trade_loader.py index 4d4b07d..c822643 100644 --- a/src/pote/ingestion/trade_loader.py +++ b/src/pote/ingestion/trade_loader.py @@ -45,27 +45,25 @@ class TradeLoader: for txn in transactions: try: - # Get or create official - official, is_new_official = self._get_or_create_official(txn) - if is_new_official: - officials_created += 1 + with self.session.begin_nested(): + official, is_new_official = self._get_or_create_official(txn) + if is_new_official: + officials_created += 1 - # Get or create security - ticker = txn.get("ticker", "").strip().upper() - if not ticker or ticker in ("N/A", "--", ""): - logger.debug( - f"Skipping transaction with no ticker: {txn.get('representative')}" - ) - continue + ticker = txn.get("ticker", "").strip().upper() + if not ticker or ticker in ("N/A", "--", ""): + logger.debug( + f"Skipping transaction with no ticker: {txn.get('representative')}" + ) + continue - security, is_new_security = self._get_or_create_security(ticker) - if is_new_security: - securities_created += 1 + security, is_new_security = self._get_or_create_security(ticker) + if is_new_security: + securities_created += 1 - # Create trade (upsert) - trade_created = self._upsert_trade(txn, official.id, security.id, source) - if trade_created: - trades_created += 1 + trade_created = self._upsert_trade(txn, official.id, security.id, source) + if trade_created: + trades_created += 1 except Exception as e: logger.error(f"Failed to ingest transaction {txn}: {e}") -- 2.49.1 From 8dd354c7c578922b198cd6e34c995453ebfa0db3 Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 26 May 2026 20:00:48 -0400 Subject: [PATCH 2/4] Fix Gitea Actions CI for act_runner on homelab. Remove Python job containers so checkout@v4 can run (needs Node), install deps on ubuntu-latest for security-scan, and stop excluding README.md from Docker build context. --- .dockerignore | 3 +-- .github/workflows/ci.yml | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.dockerignore b/.dockerignore index f6a5e40..2eed044 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,7 +45,6 @@ logs/ .DS_Store Thumbs.db -# Docs (optional - include if you want them in container) +# Docs (keep README.md โ€” required by pyproject.toml / Docker build) docs/ -*.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df8c9ad..99009a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,8 @@ on: jobs: lint-and-test: runs-on: ubuntu-latest - container: - image: python:3.11-bullseye - + # No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images) + services: postgres: image: postgres:15 @@ -31,8 +30,8 @@ jobs: - name: Install system dependencies run: | - apt-get update - apt-get install -y postgresql-client + sudo apt-get update + sudo apt-get install -y postgresql-client python3.11 python3.11-venv python3-pip - name: Install Python dependencies run: | @@ -70,27 +69,25 @@ jobs: security-scan: runs-on: ubuntu-latest - container: - image: python:3.11-bullseye steps: - name: Check out code uses: actions/checkout@v4 - name: Install dependencies run: | - pip install --upgrade pip - pip install safety bandit + python3.11 -m pip install --upgrade pip + python3.11 -m pip install safety bandit - name: Run safety check run: | - pip install -e . - safety check --json || true + python3.11 -m pip install -e . + python3.11 -m safety check --json || true continue-on-error: true - name: Run bandit security scan run: | - bandit -r src/ -f json -o bandit-report.json || true - bandit -r src/ -f screen + python3.11 -m bandit -r src/ -f json -o bandit-report.json || true + python3.11 -m bandit -r src/ -f screen continue-on-error: true dependency-scan: -- 2.49.1 From 648d5ac7420a3fc5b79022eaf570ed6838b60e78 Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 26 May 2026 20:07:49 -0400 Subject: [PATCH 3/4] Fix CI for Gitea act_runner: python3 and local docker build. Use python3 instead of unavailable python3.11 packages, drop apt step that failed on runner image, and replace buildx with docker build so the test step can run the built image. --- .github/workflows/ci.yml | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99009a4..d8fd7c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,15 +28,10 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y postgresql-client python3.11 python3.11-venv python3-pip - - name: Install Python dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" + python3 -m pip install --upgrade pip + python3 -m pip install -e ".[dev]" - name: Run linters run: | @@ -56,7 +51,7 @@ jobs: SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }} FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }} run: | - pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml + python3 -m pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml - name: Test scripts env: @@ -75,19 +70,19 @@ jobs: - name: Install dependencies run: | - python3.11 -m pip install --upgrade pip - python3.11 -m pip install safety bandit + python3 -m pip install --upgrade pip + python3 -m pip install safety bandit - name: Run safety check run: | - python3.11 -m pip install -e . - python3.11 -m safety check --json || true + python3 -m pip install -e . + python3 -m safety check --json || true continue-on-error: true - name: Run bandit security scan run: | - python3.11 -m bandit -r src/ -f json -o bandit-report.json || true - python3.11 -m bandit -r src/ -f screen + python3 -m bandit -r src/ -f json -o bandit-report.json || true + python3 -m bandit -r src/ -f screen continue-on-error: true dependency-scan: @@ -111,20 +106,10 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: false - tags: pote:test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Test Docker image + - name: Build and test Docker image run: | + # Plain docker build โ€” buildx images are not visible to act_runner's docker run + docker build -t pote:test . docker run --rm pote:test python -c "import pote; print('POTE import successful')" workflow-summary: -- 2.49.1 From bc68f8a752bdd2f0efd69f2455a7ad0857dd25bb Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 26 May 2026 20:17:56 -0400 Subject: [PATCH 4/4] Use project venv in CI to satisfy PEP 668 on Ubuntu runner. Gitea act_runner images block system-wide pip; install deps into .venv and invoke tools from .venv/bin. --- .github/workflows/ci.yml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8fd7c0..76a8590 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,19 +28,20 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Install Python dependencies + - name: Set up Python venv run: | - python3 -m pip install --upgrade pip - python3 -m pip install -e ".[dev]" + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -e ".[dev]" - name: Run linters run: | echo "Running ruff..." - ruff check src/ tests/ || true + .venv/bin/ruff check src/ tests/ || true echo "Running black check..." - black --check src/ tests/ || true + .venv/bin/black --check src/ tests/ || true echo "Running mypy..." - mypy src/ --install-types --non-interactive || true + .venv/bin/mypy src/ --install-types --non-interactive || true - name: Run tests with coverage env: @@ -51,16 +52,16 @@ jobs: SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }} FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }} run: | - python3 -m pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml + .venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml - name: Test scripts env: DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test run: | echo "Testing database migrations..." - alembic upgrade head + .venv/bin/alembic upgrade head echo "Testing price loader..." - python scripts/fetch_sample_prices.py || true + .venv/bin/python scripts/fetch_sample_prices.py || true security-scan: runs-on: ubuntu-latest @@ -68,21 +69,21 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Install dependencies + - name: Set up Python venv run: | - python3 -m pip install --upgrade pip - python3 -m pip install safety bandit + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -e ".[dev]" safety bandit - name: Run safety check run: | - python3 -m pip install -e . - python3 -m safety check --json || true + .venv/bin/safety check --json || true continue-on-error: true - name: Run bandit security scan run: | - python3 -m bandit -r src/ -f json -o bandit-report.json || true - python3 -m bandit -r src/ -f screen + .venv/bin/bandit -r src/ -f json -o bandit-report.json || true + .venv/bin/bandit -r src/ -f screen continue-on-error: true dependency-scan: -- 2.49.1