diff --git a/.dockerignore b/.dockerignore index f6a5e40..2eed044 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,7 +45,6 @@ logs/ .DS_Store Thumbs.db -# Docs (optional - include if you want them in container) +# Docs (keep README.md โ€” required by pyproject.toml / Docker build) docs/ -*.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df8c9ad..76a8590 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,8 @@ on: jobs: lint-and-test: runs-on: ubuntu-latest - container: - image: python:3.11-bullseye - + # No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images) + services: postgres: image: postgres:15 @@ -29,24 +28,20 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Install system dependencies + - name: Set up Python venv run: | - apt-get update - apt-get install -y postgresql-client - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install -e ".[dev]" + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -e ".[dev]" - name: Run linters run: | echo "Running ruff..." - ruff check src/ tests/ || true + .venv/bin/ruff check src/ tests/ || true echo "Running black check..." - black --check src/ tests/ || true + .venv/bin/black --check src/ tests/ || true echo "Running mypy..." - mypy src/ --install-types --non-interactive || true + .venv/bin/mypy src/ --install-types --non-interactive || true - name: Run tests with coverage env: @@ -57,40 +52,38 @@ jobs: SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }} FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }} run: | - pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml + .venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml - name: Test scripts env: DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test run: | echo "Testing database migrations..." - alembic upgrade head + .venv/bin/alembic upgrade head echo "Testing price loader..." - python scripts/fetch_sample_prices.py || true + .venv/bin/python scripts/fetch_sample_prices.py || true security-scan: runs-on: ubuntu-latest - container: - image: python:3.11-bullseye steps: - name: Check out code uses: actions/checkout@v4 - - name: Install dependencies + - name: Set up Python venv run: | - pip install --upgrade pip - pip install safety bandit + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -e ".[dev]" safety bandit - name: Run safety check run: | - pip install -e . - safety check --json || true + .venv/bin/safety check --json || true continue-on-error: true - name: Run bandit security scan run: | - bandit -r src/ -f json -o bandit-report.json || true - bandit -r src/ -f screen + .venv/bin/bandit -r src/ -f json -o bandit-report.json || true + .venv/bin/bandit -r src/ -f screen continue-on-error: true dependency-scan: @@ -114,20 +107,10 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: false - tags: pote:test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Test Docker image + - name: Build and test Docker image run: | + # Plain docker build โ€” buildx images are not visible to act_runner's docker run + docker build -t pote:test . docker run --rm pote:test python -c "import pote; print('POTE import successful')" workflow-summary: diff --git a/EMAIL_SETUP.md b/EMAIL_SETUP.md index 653d9fc..d917e94 100644 --- a/EMAIL_SETUP.md +++ b/EMAIL_SETUP.md @@ -1,18 +1,18 @@ -# Email Setup for levkin.ca +# Email Setup for levkine.ca (Mailcow) -Your POTE system is configured to use `test@levkin.ca` for sending reports. +Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`. ## โœ… Configuration Done The `.env` file has been created with these settings: ```env -SMTP_HOST=mail.levkin.ca +SMTP_HOST=10.0.10.132 SMTP_PORT=587 -SMTP_USER=test@levkin.ca +SMTP_USER=alerts@levkine.ca SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE -FROM_EMAIL=test@levkin.ca -REPORT_RECIPIENTS=test@levkin.ca +FROM_EMAIL=alerts@levkine.ca +REPORT_RECIPIENTS=idobkin@gmail.com ``` ## ๐Ÿ”‘ Next Steps diff --git a/pyproject.toml b/pyproject.toml index 44473ff..8259445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,10 @@ dependencies = [ "pydantic-settings>=2.0", "python-dotenv>=1.0", "requests>=2.31", + "httpx>=0.27", "pandas>=2.0", "numpy>=1.24", + "scikit-learn>=1.3", "yfinance>=0.2", "psycopg2-binary>=2.9", ] diff --git a/scripts/fetch_congressional_trades.py b/scripts/fetch_congressional_trades.py index da8fcfd..33bb4e6 100755 --- a/scripts/fetch_congressional_trades.py +++ b/scripts/fetch_congressional_trades.py @@ -28,8 +28,8 @@ def main(): args = parser.parse_args() - logger.info("=== Fetching Congressional Trades from House Stock Watcher ===") - logger.info("Source: https://housestockwatcher.com (free, no API key)") + logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===") + logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)") try: with HouseWatcherClient() as client: diff --git a/src/pote/config.py b/src/pote/config.py index b35e05c..5abce11 100644 --- a/src/pote/config.py +++ b/src/pote/config.py @@ -30,6 +30,16 @@ class Settings(BaseSettings): # Logging log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)") + # Email (Mailcow @ mail.levkine.ca โ€” use LAN IP from app LXCs if DNS points public) + smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname") + smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)") + smtp_user: str = Field(default="", description="SMTP auth username") + smtp_password: str = Field(default="", description="SMTP auth password") + from_email: str = Field(default="", description="From address for reports") + report_recipients: str = Field( + default="", description="Default report recipients (comma-separated)" + ) + # Application app_name: str = "POTE" app_version: str = "0.1.0" diff --git a/src/pote/db/__init__.py b/src/pote/db/__init__.py index aa3e296..b547fbb 100644 --- a/src/pote/db/__init__.py +++ b/src/pote/db/__init__.py @@ -3,6 +3,7 @@ Database layer: engine, session factory, and base model. """ from collections.abc import Generator +from contextlib import contextmanager from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker @@ -26,8 +27,9 @@ class Base(DeclarativeBase): pass +@contextmanager def get_session() -> Generator[Session, None, None]: - """Get a database session (use as a context manager or dependency).""" + """Get a database session (context manager or FastAPI-style dependency).""" session = SessionLocal() try: yield session diff --git a/src/pote/ingestion/house_watcher.py b/src/pote/ingestion/house_watcher.py index 7892532..ee5e745 100644 --- a/src/pote/ingestion/house_watcher.py +++ b/src/pote/ingestion/house_watcher.py @@ -1,9 +1,12 @@ """ House Stock Watcher client for fetching congressional trade data. -Free, no API key required - scrapes from housestockwatcher.com + +Uses public STOCK Act disclosure datasets (no API key). The legacy +housestockwatcher.com host is often unavailable; we try several mirrors. """ import logging +import os from datetime import date, datetime from typing import Any @@ -11,16 +14,69 @@ import httpx logger = logging.getLogger(__name__) +def _default_data_urls() -> tuple[str, ...]: + override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip() + if override: + return (override,) + return ( + "https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json", + "https://housestockwatcher.com/api/all_transactions", + "https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json", + ) + + +DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls() + + +def _ascii_safe(text: str | None) -> str: + """Normalize text for DB fields that may use ASCII-only client encoding.""" + if not text: + return "" + return text.replace("\u00b7", "-").replace("ยท", "-").strip() + + +def _party_label(code: str | None) -> str: + if not code: + return "" + mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"} + return mapping.get(code.strip().upper(), code.strip()) + + +def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]: + """ + Map a raw disclosure record to the House Stock Watcher field names + expected by TradeLoader. + """ + if "representative" in raw and "disclosure_date" in raw: + return raw + + # Kadoa congress-trading-monitor export + if "filer_name" in raw: + chamber = (raw.get("chamber") or "").strip().lower() + house = "Senate" if chamber == "senate" else "House" + return { + "representative": raw.get("filer_name", "").strip(), + "ticker": (raw.get("ticker") or "").strip(), + "transaction_date": raw.get("transaction_date", ""), + "disclosure_date": raw.get("filing_date", ""), + "transaction": raw.get("transaction_type", ""), + "amount": raw.get("amount_range_label", ""), + "house": house, + "district": _ascii_safe(raw.get("office")), + "party": _party_label(raw.get("party")), + } + + return raw + class HouseWatcherClient: """ - Client for House Stock Watcher API (free, community-maintained). + Client for congressional trade JSON feeds (free, community-maintained). - Data source: https://housestockwatcher.com/ - No authentication required. + Primary source: congress-trading-monitor public dataset on GitHub. """ - BASE_URL = "https://housestockwatcher.com/api" + data_urls: tuple[str, ...] = DEFAULT_DATA_URLS def __init__(self, timeout: float = 30.0): """ @@ -65,30 +121,31 @@ class HouseWatcherClient: Raises: httpx.HTTPError: If request fails """ - url = f"{self.BASE_URL}/all_transactions" - logger.info(f"Fetching transactions from {url}") + last_error: Exception | None = None + for url in self.data_urls: + if not url: + continue + logger.info(f"Fetching transactions from {url}") + try: + response = self._client.get(url) + response.raise_for_status() + data = response.json() + if not isinstance(data, list): + raise ValueError(f"Expected list response, got {type(data)}") + data = [normalize_transaction_record(item) for item in data] + logger.info(f"Fetched {len(data)} transactions from {url}") + if limit: + data = data[:limit] + return data + except Exception as e: + last_error = e + logger.warning(f"Failed to fetch from {url}: {e}") + continue - try: - response = self._client.get(url) - response.raise_for_status() - data = response.json() - - if not isinstance(data, list): - raise ValueError(f"Expected list response, got {type(data)}") - - logger.info(f"Fetched {len(data)} transactions from House Stock Watcher") - - if limit: - data = data[:limit] - - return data - - except httpx.HTTPError as e: - logger.error(f"Failed to fetch from House Stock Watcher: {e}") - raise - except Exception as e: - logger.error(f"Unexpected error fetching transactions: {e}") - raise + logger.error("Failed to fetch congressional trades from all configured URLs") + if last_error: + raise last_error + raise RuntimeError("No data URLs configured") def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]: """ diff --git a/src/pote/ingestion/trade_loader.py b/src/pote/ingestion/trade_loader.py index 4d4b07d..c822643 100644 --- a/src/pote/ingestion/trade_loader.py +++ b/src/pote/ingestion/trade_loader.py @@ -45,27 +45,25 @@ class TradeLoader: for txn in transactions: try: - # Get or create official - official, is_new_official = self._get_or_create_official(txn) - if is_new_official: - officials_created += 1 + with self.session.begin_nested(): + official, is_new_official = self._get_or_create_official(txn) + if is_new_official: + officials_created += 1 - # Get or create security - ticker = txn.get("ticker", "").strip().upper() - if not ticker or ticker in ("N/A", "--", ""): - logger.debug( - f"Skipping transaction with no ticker: {txn.get('representative')}" - ) - continue + ticker = txn.get("ticker", "").strip().upper() + if not ticker or ticker in ("N/A", "--", ""): + logger.debug( + f"Skipping transaction with no ticker: {txn.get('representative')}" + ) + continue - security, is_new_security = self._get_or_create_security(ticker) - if is_new_security: - securities_created += 1 + security, is_new_security = self._get_or_create_security(ticker) + if is_new_security: + securities_created += 1 - # Create trade (upsert) - trade_created = self._upsert_trade(txn, official.id, security.id, source) - if trade_created: - trades_created += 1 + trade_created = self._upsert_trade(txn, official.id, security.id, source) + if trade_created: + trades_created += 1 except Exception as e: logger.error(f"Failed to ingest transaction {txn}: {e}")