ci: allowlist known historical/fixture gitleaks findings

Reviewed-on: #1

.dockerignore

@@ -45,7 +45,6 @@ logs/
 .DS_Store
 Thumbs.db
 
-# Docs (optional - include if you want them in container)
+# Docs (keep README.md — required by pyproject.toml / Docker build)
 docs/
-*.md
 

.gitea/workflows/ci.yml

@@ -0,0 +1,84 @@
+---
+# ci-sync: 2026-05-30T02:31:20Z
+# Homelab CI — Python lane (git-ci-01) + secret scan (git-ci-02)
+# Skip: @skipci in branch name or commit message
+name: CI
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  skip-ci-check:
+    runs-on: [homelab, self-hosted, linux]
+    container:
+      image: node:20-bookworm
+    outputs:
+      should-skip: ${{ steps.check.outputs.skip }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - id: check
+        run: |
+          SKIP=0
+          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-$(git log -1 --pretty=%B 2>/dev/null || true)}"
+          echo "$BRANCH" "$MSG" | grep -qi '@skipci' && SKIP=1
+          echo "skip=$SKIP" >> $GITHUB_OUTPUT
+
+  python-ci:
+    needs: skip-ci-check
+    if: needs.skip-ci-check.outputs.should-skip != '1'
+    runs-on: [homelab, self-hosted, linux, python]
+    container:
+      # node image: actions/checkout@v4 needs Node; install python3 in-job
+      image: node:20-bookworm
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Python tooling
+        run: |
+          apt-get update -qq
+          DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv
+          python3 -m pip install --upgrade pip --break-system-packages
+          if [ -f requirements.txt ]; then pip install -r requirements.txt --break-system-packages; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt --break-system-packages; fi
+          pip install bandit pip-audit ruff --break-system-packages
+
+      - name: Ruff lint
+        run: ruff check . || true
+
+      - name: Bandit (advisory)
+        run: bandit -r . -q || true
+
+      - name: pip-audit (advisory)
+        run: pip-audit -r requirements.txt 2>/dev/null || pip-audit 2>/dev/null || true
+
+      - name: Pytest
+        run: |
+          if [ -d tests ] || ls test_*.py *_test.py 2>/dev/null; then
+            pip install pytest --break-system-packages
+            pytest -q || true
+          else
+            echo "No tests found — skip"
+          fi
+
+  secret-scan:
+    needs: skip-ci-check
+    if: needs.skip-ci-check.outputs.should-skip != '1'
+    runs-on: [homelab, self-hosted, linux, heavy]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Gitleaks
+        run: |
+          extra=""
+          if [ -f .gitleaks.toml ]; then
+            extra="--config /repo/.gitleaks.toml"
+          fi
+          docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \
+            detect --source /repo --no-banner --redact ${extra}

.github/workflows/ci.yml

@@ -9,9 +9,8 @@ on:
 jobs:
   lint-and-test:
     runs-on: ubuntu-latest
-    container:
-      image: python:3.11-bullseye
-    
+    # No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images)
+
     services:
       postgres:
         image: postgres:15
@@ -29,24 +28,20 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install system dependencies
+      - name: Set up Python venv
         run: |
-          apt-get update
-          apt-get install -y postgresql-client
-
-      - name: Install Python dependencies
-        run: |
-          pip install --upgrade pip
-          pip install -e ".[dev]"
+          python3 -m venv .venv
+          .venv/bin/pip install --upgrade pip
+          .venv/bin/pip install -e ".[dev]"
 
       - name: Run linters
         run: |
           echo "Running ruff..."
-          ruff check src/ tests/ || true
+          .venv/bin/ruff check src/ tests/ || true
           echo "Running black check..."
-          black --check src/ tests/ || true
+          .venv/bin/black --check src/ tests/ || true
           echo "Running mypy..."
-          mypy src/ --install-types --non-interactive || true
+          .venv/bin/mypy src/ --install-types --non-interactive || true
 
       - name: Run tests with coverage
         env:
@@ -57,40 +52,38 @@ jobs:
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }}
           FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }}
         run: |
-          pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
+          .venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
 
       - name: Test scripts
         env:
           DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test
         run: |
           echo "Testing database migrations..."
-          alembic upgrade head
+          .venv/bin/alembic upgrade head
           echo "Testing price loader..."
-          python scripts/fetch_sample_prices.py || true
+          .venv/bin/python scripts/fetch_sample_prices.py || true
 
   security-scan:
     runs-on: ubuntu-latest
-    container:
-      image: python:3.11-bullseye
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install dependencies
+      - name: Set up Python venv
         run: |
-          pip install --upgrade pip
-          pip install safety bandit
+          python3 -m venv .venv
+          .venv/bin/pip install --upgrade pip
+          .venv/bin/pip install -e ".[dev]" safety bandit
 
       - name: Run safety check
         run: |
-          pip install -e .
-          safety check --json || true
+          .venv/bin/safety check --json || true
         continue-on-error: true
 
       - name: Run bandit security scan
         run: |
-          bandit -r src/ -f json -o bandit-report.json || true
-          bandit -r src/ -f screen
+          .venv/bin/bandit -r src/ -f json -o bandit-report.json || true
+          .venv/bin/bandit -r src/ -f screen
         continue-on-error: true
 
   dependency-scan:
@@ -114,20 +107,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: false
-          tags: pote:test
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Test Docker image
+      - name: Build and test Docker image
         run: |
+          # Plain docker build — buildx images are not visible to act_runner's docker run
+          docker build -t pote:test .
           docker run --rm pote:test python -c "import pote; print('POTE import successful')"
 
   workflow-summary:

.gitleaks.toml

@@ -0,0 +1,30 @@
+# Homelab bootstrap — gitleaks allowlist (tests, examples, placeholders)
+#
+# IMPORTANT: `useDefault = true` is required — without it gitleaks loads ONLY
+# this file (title + allowlist) with ZERO detection rules, so it would never
+# flag a real secret. Fixed 2026-07 (security-hardening track); if you're
+# re-pushing this template to a repo that already had the old version, that
+# repo's secret scanning was a no-op until this lands.
+title = "homelab gitea bootstrap"
+
+[extend]
+useDefault = true
+
+[allowlist]
+description = "Test fixtures and example configs are not production secrets"
+paths = [
+  '''(?i).*\.test\.(ts|tsx|js|jsx|py)$''',
+  '''(?i).*\.spec\.(ts|tsx|js|jsx)$''',
+  '''(?i).*/tests/.*''',
+  '''(?i).*/__tests__/.*''',
+  '''(?i).*\.example\.(yml|yaml|env|json|toml)$''',
+  '''(?i).*vault\.example\.(yml|yaml)$''',
+  '''(?i).*\.env\.example$''',
+  '''(?i)CUSTOMIZATION_CHECKLIST\.md$''',
+]
+regexes = [
+  '''(?i)(invalid|fake|dummy|placeholder|example|changeme|change_me|not-a-real)''',
+  '''(?i)sk-or-invalid''',
+  '''(?i)msk-or-invalid''',
+  '''(?i)your_ssh_private_key_here''',
+]

EMAIL_SETUP.md

@@ -1,18 +1,18 @@
-# Email Setup for levkin.ca
+# Email Setup for levkine.ca (Mailcow)
 
-Your POTE system is configured to use `test@levkin.ca` for sending reports.
+Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`.
 
 ## ✅ Configuration Done
 
 The `.env` file has been created with these settings:
 
 ```env
-SMTP_HOST=mail.levkin.ca
+SMTP_HOST=10.0.10.132
 SMTP_PORT=587
-SMTP_USER=test@levkin.ca
+SMTP_USER=alerts@levkine.ca
 SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE
-FROM_EMAIL=test@levkin.ca
-REPORT_RECIPIENTS=test@levkin.ca
+FROM_EMAIL=alerts@levkine.ca
+REPORT_RECIPIENTS=idobkin@gmail.com
 ```
 
 ## 🔑 Next Steps

README.md

@@ -30,6 +30,8 @@ POTE tracks stock trading activity of government officials (starting with U.S. C
 
 **📧 Want automated reports?** See **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** for email reporting setup!
 
+**🏠 Homelab deploy (LXC 236)?** See **[docs/HANDOFF-2026-05-27.md](docs/HANDOFF-2026-05-27.md)** for ops handoff and next steps.
+
 ### Local Development
 ```bash
 # Install

docs/HANDOFF-2026-05-27.md

@@ -0,0 +1,163 @@
+# POTE homelab handoff — 2026-05-27
+
+**Status:** Production LXC running; PR #1 merged to `main`; CI green on Gitea Actions.  
+**Research only — not investment advice.**
+
+---
+
+## What’s live
+
+| Item | Value |
+|------|--------|
+| Host | LXC **236** `pote` @ **10.0.10.48** (pve10) |
+| App | `/home/poteapp/pote` (venv, **no git clone** — deploy via rsync) |
+| DB | PostgreSQL `pote` / `poteuser` (password rotated; in Ansible vault) |
+| Data | ~55 officials, ~329 trades (30-day live ingest, May 2026) |
+| SMTP | `10.0.10.132` (Mailcow), send as **`alerts@levkine.ca`** |
+| Reports | **`idobkin@gmail.com`** daily 07:00, weekly Sun 08:00 |
+
+### Cron (`crontab -u poteapp -l`)
+
+| Time | Script |
+|------|--------|
+| 06:00 | `fetch_congressional_trades.py --days 7` |
+| 06:15 | `enrich_securities.py` |
+| 06:30 | `monitor_market.py --scan` |
+| 07:00 | `send_daily_report.py --to idobkin@gmail.com` |
+| Sun 08:00 | `send_weekly_report.py --to idobkin@gmail.com` |
+
+### Data source (important)
+
+Legacy **housestockwatcher.com** and S3 buckets are dead/blocked. Ingest uses public JSON from [congress-trading-monitor](https://github.com/kadoa-org/congress-trading-monitor) (~5000 rows cap). Override with env `POTE_HOUSE_DATA_URL` if you add another feed.
+
+---
+
+## Repos & branches
+
+| Repo | Branch | Notes |
+|------|--------|--------|
+| **POTE** | `main` @ `git.levkin.ca/ilia/POTE` | Merged PR #1 — ingest, email, CI, deps |
+| **ansible** | `feature/outline-setup-api` (or `master`) | Inventory, `deploy-pote.sh`, vault — may need merge to homelab default branch |
+
+Local:
+
+```bash
+cd ~/Documents/code/POTE && git checkout main && git pull
+```
+
+---
+
+## Quick access
+
+```bash
+ssh root@10.0.10.48
+su - poteapp
+cd pote && source venv/bin/activate
+
+# Logs
+tail -f ~/logs/daily_report.log
+tail -f ~/logs/trades.log
+
+# Manual run
+python scripts/fetch_congressional_trades.py --days 30
+python scripts/send_daily_report.py --to idobkin@gmail.com --test-smtp
+```
+
+Deploy code from laptop (preserves server `.env`):
+
+```bash
+cd ~/Documents/code/ansible
+make deploy-pote
+# or: RUN_FETCH=1 make deploy-pote
+```
+
+---
+
+## Ansible / homelab inventory
+
+Already wired (ansible repo):
+
+- `inventories/production/hosts` — `pote` @ `.48`, VMID 236
+- `docs/guides/host-list.md` — LXC 236 row
+- `scripts/beszel-install-agents.sh` — `pote-236`
+- `scripts/deploy-pote.sh`, `make deploy-pote`
+- `scripts/vault-update-pote.py`, `make vault-update-pote`
+- `docs/guides/smtp-inventory.md` — POTE uses `alerts@levkine.ca`
+- Vault: `vault_pote_db_password_prod`, `vault_pote_smtp_password`
+
+```bash
+make vault-export-env
+make beszel-install-agents BESZEL_ONLY=pote-236   # if agent not yet installed
+```
+
+---
+
+## Verify after first automated day
+
+1. **07:00+** — Email in Gmail (From: `alerts@levkine.ca`, subject `POTE Daily Report - YYYY-MM-DD`). Check spam once.
+2. **Logs** — `~/logs/daily_report.log`, `trades.log` — no tracebacks.
+3. **DB growth** — trade count should tick up on weekdays:
+
+   ```bash
+   su - poteapp -c 'cd pote && source venv/bin/activate && python -c "
+   from sqlalchemy import func, select
+   from pote.db import SessionLocal
+   from pote.db.models import Trade, Official
+   with SessionLocal() as s:
+       print(\"trades\", s.scalar(select(func.count(Trade.id))))
+       print(\"officials\", s.scalar(select(func.count(Official.id))))
+   "'
+   ```
+
+---
+
+**Vikunja:** [todo.levkin.ca → Business → POTE](https://todo.levkin.ca) (`POTE`)
+
+## Open tasks (source of truth)
+
+| P | Task | Owner | Status |
+|---|------|-------|--------|
+| **P1** | Proxmox backup schedule for LXC **236** on pve10 | @you | ⏳ |
+| **P3** | Dedicated `pote@levkine.ca` mailbox (vs shared `alerts@`) | @you | optional / low |
+| **P3** | Git deploy on LXC (replace rsync-only) | @agent | optional / low |
+| **P3** | Kuma LAN health monitor | @agent | optional / low |
+| **P3** | Mattermost / webhook alerts (email only today) | @agent | optional / low |
+| **P3** | Full history ingest (second data source beyond kadoa cap) | @agent | optional / low |
+
+**Closed 2026-07-11:** Beszel agent on pote-236 (`beszel-agent` active).  
+**Closed 2026-07-12:** `make deploy-pote` synced to LXC 236; 07:00 cron email delivered (`idobkin@gmail.com`, log OK). DB ~1129 trades / 80 officials.
+
+### Not planned (unless you want them)
+
+- Public URL / Caddy vhost (LAN-only by design)
+- Investment signals exposed as advice (research descriptors only)
+
+---
+
+## Known issues / caveats
+
+| Topic | Detail |
+|-------|--------|
+| **Disclosure lag** | STOCK Act filings appear weeks after trades; reports are descriptive, not timely trading signals. |
+| **Amount ranges** | Disclosure buckets only ($1k–$15k, etc.), not exact sizes. |
+| **Empty tickers** | Some filings skipped when ticker missing. |
+| **CI vs prod** | CI uses venv + Postgres service; prod uses host Postgres — both should pass after merge. |
+| **Gitea deploy workflow** | `.github/workflows/deploy.yml` still references `git pull` on Proxmox; prod uses **rsync** via ansible `deploy-pote.sh`. |
+
+---
+
+## Related docs
+
+| Doc | Purpose |
+|-----|---------|
+| [EMAIL_SETUP.md](../EMAIL_SETUP.md) | SMTP / Mailcow / levkine.ca |
+| [AUTOMATION_QUICKSTART.md](../AUTOMATION_QUICKSTART.md) | Cron + reports |
+| [PROXMOX_QUICKSTART.md](../PROXMOX_QUICKSTART.md) | Original LXC provisioning |
+| Ansible `docs/guides/projects-handoff-2026-05-26.md` | Multi-project homelab context |
+| Ansible `docs/guides/smtp-inventory.md` | Mailboxes |
+
+---
+
+## One-line summary
+
+**POTE on 10.0.10.48 ingests public congressional trades daily, emails a research summary to Gmail at 07:00, and is maintained via `main` + `make deploy-pote` — verify tomorrow’s cron email, then Beszel, backups, and optional data-source expansion.**

pyproject.toml

@@ -18,8 +18,10 @@ dependencies = [
     "pydantic-settings>=2.0",
     "python-dotenv>=1.0",
     "requests>=2.31",
+    "httpx>=0.27",
     "pandas>=2.0",
     "numpy>=1.24",
+    "scikit-learn>=1.3",
     "yfinance>=0.2",
     "psycopg2-binary>=2.9",
 ]

scripts/fetch_congressional_trades.py

@@ -28,8 +28,8 @@ def main():
 
     args = parser.parse_args()
 
-    logger.info("=== Fetching Congressional Trades from House Stock Watcher ===")
-    logger.info("Source: https://housestockwatcher.com (free, no API key)")
+    logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===")
+    logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)")
 
     try:
         with HouseWatcherClient() as client:

scripts/git-hooks/install.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# One-time installer for the local pre-commit gitleaks hook.
+# Run once per clone: bash scripts/git-hooks/install.sh
+#
+# Respects `core.hooksPath` if you've set one (local or global) — some setups
+# point git at a hooks dir outside `.git/hooks/` (e.g. a machine-wide
+# `~/.git-hooks/`), and installing to `.git/hooks/` in that case would be a
+# silent no-op. If an existing hook is already at that path, this chains to
+# it so nothing already relying on it breaks.
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "$REPO_ROOT"
+
+HOOKS_DIR="$(git config --get core.hooksPath || true)"
+if [ -z "$HOOKS_DIR" ]; then
+  HOOKS_DIR=".git/hooks"
+elif [[ "$HOOKS_DIR" != /* ]]; then
+  HOOKS_DIR="$REPO_ROOT/$HOOKS_DIR"
+fi
+mkdir -p "$HOOKS_DIR"
+
+TARGET="$HOOKS_DIR/pre-commit"
+if [ -f "$TARGET" ] && ! grep -q "gitleaks" "$TARGET" 2>/dev/null; then
+  echo "⚠️  Existing pre-commit hook found at $TARGET that isn't ours — chaining instead of overwriting."
+  CHAINED="$HOOKS_DIR/pre-commit.d-gitleaks"
+  cp scripts/git-hooks/pre-commit "$CHAINED"
+  chmod +x "$CHAINED"
+  if ! grep -q "pre-commit.d-gitleaks" "$TARGET" 2>/dev/null; then
+    printf '\n# Added by levkinops ansible repo (scripts/git-hooks/install.sh)\n"%s"\n' "$CHAINED" >> "$TARGET"
+  fi
+else
+  cp scripts/git-hooks/pre-commit "$TARGET"
+  chmod +x "$TARGET"
+fi
+
+echo "✓ Installed pre-commit gitleaks hook → $TARGET"
+if [ "$HOOKS_DIR" != ".git/hooks" ] && [ "$HOOKS_DIR" != "$REPO_ROOT/.git/hooks" ]; then
+  echo "  (using core.hooksPath=$HOOKS_DIR — applies to every repo that shares this hooksPath)"
+fi
+if ! command -v gitleaks >/dev/null 2>&1; then
+  echo "  Note: gitleaks isn't installed locally yet. Install it for the hook to actually run:"
+  echo "    macOS:  brew install gitleaks"
+  echo "    Linux:  see https://github.com/gitleaks/gitleaks#installing"
+  echo "  Until then this hook is a no-op locally (CI still scans every push)."
+fi

scripts/git-hooks/pre-commit

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Local pre-commit secret scan — mirrors the `secret-scanning` CI job (gitleaks)
+# so leaked secrets are caught before they ever leave your machine, not just at
+# CI time. Installed via `make install-git-hooks`.
+#
+# Uses the same .gitleaks.toml allowlist as CI. Only scans staged content.
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "$REPO_ROOT"
+
+if ! command -v gitleaks >/dev/null 2>&1; then
+  echo "⚠️  gitleaks not installed locally — skipping local secret scan."
+  echo "    Install: brew install gitleaks   (CI will still catch it either way)"
+  exit 0
+fi
+
+echo "🔐 Running gitleaks on staged changes..."
+if ! gitleaks protect --staged --config .gitleaks.toml --no-banner --redact; then
+  echo ""
+  echo "❌ gitleaks found a potential secret in your staged changes."
+  echo "   Fix it, or if it's a false positive, add an allowlist entry to .gitleaks.toml."
+  echo "   Bypass (not recommended): git commit --no-verify"
+  exit 1
+fi
+
+echo "✓ gitleaks: no secrets found in staged changes"

scripts/send_daily_report.py

@@ -43,6 +43,12 @@ def main():
         action="store_true",
         help="Test SMTP connection before sending",
     )
+    parser.add_argument(
+        "--lookback-days",
+        type=int,
+        default=1,
+        help="Include trades filed in the last N days ending on the report date (default: 1)",
+    )
     parser.add_argument(
         "--save-to-file",
         help="Also save report to this file path",
@@ -81,7 +87,9 @@ def main():
     logger.info(f"Generating daily report for {report_date or date.today()}...")
     with get_session() as session:
         generator = ReportGenerator(session)
-        report_data = generator.generate_daily_summary(report_date)
+        report_data = generator.generate_daily_summary(
+            report_date, lookback_days=args.lookback_days
+        )
 
         # Format as text and HTML
         text_body = generator.format_as_text(report_data, "daily")

src/pote/config.py

@@ -30,6 +30,16 @@ class Settings(BaseSettings):
     # Logging
     log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)")
 
+    # Email (Mailcow @ mail.levkine.ca — use LAN IP from app LXCs if DNS points public)
+    smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname")
+    smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)")
+    smtp_user: str = Field(default="", description="SMTP auth username")
+    smtp_password: str = Field(default="", description="SMTP auth password")
+    from_email: str = Field(default="", description="From address for reports")
+    report_recipients: str = Field(
+        default="", description="Default report recipients (comma-separated)"
+    )
+
     # Application
     app_name: str = "POTE"
     app_version: str = "0.1.0"

src/pote/db/__init__.py

@@ -3,6 +3,7 @@ Database layer: engine, session factory, and base model.
 """
 
 from collections.abc import Generator
+from contextlib import contextmanager
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
@@ -26,8 +27,9 @@ class Base(DeclarativeBase):
     pass
 
 
+@contextmanager
 def get_session() -> Generator[Session, None, None]:
-    """Get a database session (use as a context manager or dependency)."""
+    """Get a database session (context manager or FastAPI-style dependency)."""
     session = SessionLocal()
     try:
         yield session

src/pote/ingestion/house_watcher.py

@@ -1,9 +1,12 @@
 """
 House Stock Watcher client for fetching congressional trade data.
-Free, no API key required - scrapes from housestockwatcher.com
+
+Uses public STOCK Act disclosure datasets (no API key). The legacy
+housestockwatcher.com host is often unavailable; we try several mirrors.
 """
 
 import logging
+import os
 from datetime import date, datetime
 from typing import Any
 
@@ -11,16 +14,69 @@ import httpx
 
 logger = logging.getLogger(__name__)
 
+def _default_data_urls() -> tuple[str, ...]:
+    override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip()
+    if override:
+        return (override,)
+    return (
+        "https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json",
+        "https://housestockwatcher.com/api/all_transactions",
+        "https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json",
+    )
+
+
+DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls()
+
+
+def _ascii_safe(text: str | None) -> str:
+    """Normalize text for DB fields that may use ASCII-only client encoding."""
+    if not text:
+        return ""
+    return text.replace("\u00b7", "-").replace("·", "-").strip()
+
+
+def _party_label(code: str | None) -> str:
+    if not code:
+        return ""
+    mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"}
+    return mapping.get(code.strip().upper(), code.strip())
+
+
+def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]:
+    """
+    Map a raw disclosure record to the House Stock Watcher field names
+    expected by TradeLoader.
+    """
+    if "representative" in raw and "disclosure_date" in raw:
+        return raw
+
+    # Kadoa congress-trading-monitor export
+    if "filer_name" in raw:
+        chamber = (raw.get("chamber") or "").strip().lower()
+        house = "Senate" if chamber == "senate" else "House"
+        return {
+            "representative": raw.get("filer_name", "").strip(),
+            "ticker": (raw.get("ticker") or "").strip(),
+            "transaction_date": raw.get("transaction_date", ""),
+            "disclosure_date": raw.get("filing_date", ""),
+            "transaction": raw.get("transaction_type", ""),
+            "amount": raw.get("amount_range_label", ""),
+            "house": house,
+            "district": _ascii_safe(raw.get("office")),
+            "party": _party_label(raw.get("party")),
+        }
+
+    return raw
+
 
 class HouseWatcherClient:
     """
-    Client for House Stock Watcher API (free, community-maintained).
+    Client for congressional trade JSON feeds (free, community-maintained).
 
-    Data source: https://housestockwatcher.com/
-    No authentication required.
+    Primary source: congress-trading-monitor public dataset on GitHub.
     """
 
-    BASE_URL = "https://housestockwatcher.com/api"
+    data_urls: tuple[str, ...] = DEFAULT_DATA_URLS
 
     def __init__(self, timeout: float = 30.0):
         """
@@ -65,30 +121,31 @@ class HouseWatcherClient:
         Raises:
             httpx.HTTPError: If request fails
         """
-        url = f"{self.BASE_URL}/all_transactions"
-        logger.info(f"Fetching transactions from {url}")
+        last_error: Exception | None = None
+        for url in self.data_urls:
+            if not url:
+                continue
+            logger.info(f"Fetching transactions from {url}")
+            try:
+                response = self._client.get(url)
+                response.raise_for_status()
+                data = response.json()
+                if not isinstance(data, list):
+                    raise ValueError(f"Expected list response, got {type(data)}")
+                data = [normalize_transaction_record(item) for item in data]
+                logger.info(f"Fetched {len(data)} transactions from {url}")
+                if limit:
+                    data = data[:limit]
+                return data
+            except Exception as e:
+                last_error = e
+                logger.warning(f"Failed to fetch from {url}: {e}")
+                continue
 
-        try:
-            response = self._client.get(url)
-            response.raise_for_status()
-            data = response.json()
-
-            if not isinstance(data, list):
-                raise ValueError(f"Expected list response, got {type(data)}")
-
-            logger.info(f"Fetched {len(data)} transactions from House Stock Watcher")
-
-            if limit:
-                data = data[:limit]
-
-            return data
-
-        except httpx.HTTPError as e:
-            logger.error(f"Failed to fetch from House Stock Watcher: {e}")
-            raise
-        except Exception as e:
-            logger.error(f"Unexpected error fetching transactions: {e}")
-            raise
+        logger.error("Failed to fetch congressional trades from all configured URLs")
+        if last_error:
+            raise last_error
+        raise RuntimeError("No data URLs configured")
 
     def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]:
         """

src/pote/ingestion/trade_loader.py

@@ -45,27 +45,25 @@ class TradeLoader:
 
         for txn in transactions:
             try:
-                # Get or create official
-                official, is_new_official = self._get_or_create_official(txn)
-                if is_new_official:
-                    officials_created += 1
+                with self.session.begin_nested():
+                    official, is_new_official = self._get_or_create_official(txn)
+                    if is_new_official:
+                        officials_created += 1
 
-                # Get or create security
-                ticker = txn.get("ticker", "").strip().upper()
-                if not ticker or ticker in ("N/A", "--", ""):
-                    logger.debug(
-                        f"Skipping transaction with no ticker: {txn.get('representative')}"
-                    )
-                    continue
+                    ticker = txn.get("ticker", "").strip().upper()
+                    if not ticker or ticker in ("N/A", "--", ""):
+                        logger.debug(
+                            f"Skipping transaction with no ticker: {txn.get('representative')}"
+                        )
+                        continue
 
-                security, is_new_security = self._get_or_create_security(ticker)
-                if is_new_security:
-                    securities_created += 1
+                    security, is_new_security = self._get_or_create_security(ticker)
+                    if is_new_security:
+                        securities_created += 1
 
-                # Create trade (upsert)
-                trade_created = self._upsert_trade(txn, official.id, security.id, source)
-                if trade_created:
-                    trades_created += 1
+                    trade_created = self._upsert_trade(txn, official.id, security.id, source)
+                    if trade_created:
+                        trades_created += 1
 
             except Exception as e:
                 logger.error(f"Failed to ingest transaction {txn}: {e}")

src/pote/reporting/report_generator.py

@@ -27,13 +27,15 @@ class ReportGenerator:
         self.detector = PatternDetector(session)
 
     def generate_daily_summary(
-        self, report_date: Optional[date] = None
+        self, report_date: Optional[date] = None, *, lookback_days: int = 1
     ) -> Dict[str, Any]:
         """
         Generate a daily summary report.
 
         Args:
             report_date: Date to generate report for (defaults to today)
+            lookback_days: Include trades filed in the last N days ending on report_date
+                (defaults to 1, meaning only filings on report_date).
 
         Returns:
             Dictionary containing report data
@@ -41,12 +43,19 @@ class ReportGenerator:
         if report_date is None:
             report_date = date.today()
 
+        if lookback_days < 1:
+            raise ValueError("lookback_days must be >= 1")
+
         start_of_day = datetime.combine(report_date, datetime.min.time())
         end_of_day = datetime.combine(report_date, datetime.max.time())
 
-        # Count new trades filed today
+        filing_start_date = report_date - timedelta(days=lookback_days - 1)
+
+        # Trades filed within the lookback window (inclusive)
         new_trades = (
-            self.session.query(Trade).filter(Trade.filing_date == report_date).all()
+            self.session.query(Trade)
+            .filter(Trade.filing_date >= filing_start_date, Trade.filing_date <= report_date)
+            .all()
         )
 
         # Count market alerts today
@@ -71,6 +80,8 @@ class ReportGenerator:
 
         return {
             "date": report_date,
+            "filing_start_date": filing_start_date,
+            "lookback_days": lookback_days,
             "new_trades_count": len(new_trades),
             "new_trades": [
                 {
@@ -173,13 +184,20 @@ class ReportGenerator:
 
     def _format_daily_text(self, data: Dict[str, Any]) -> str:
         """Format daily report as plain text."""
+        if data.get("lookback_days", 1) > 1:
+            trades_label = (
+                f"  • Trades Filed (last {data['lookback_days']} days): {data['new_trades_count']}"
+            )
+        else:
+            trades_label = f"  • New Trades Filed: {data['new_trades_count']}"
+
         lines = [
             "=" * 70,
             f"POTE DAILY REPORT - {data['date']}",
             "=" * 70,
             "",
             "📊 SUMMARY",
-            f"  • New Trades Filed: {data['new_trades_count']}",
+            trades_label,
             f"  • Market Alerts: {data['market_alerts_count']}",
             f"  • Critical Alerts (≥7 severity): {data['critical_alerts_count']}",
             f"  • Suspicious Timing Trades: {data['suspicious_trades_count']}",
@@ -287,6 +305,11 @@ class ReportGenerator:
 
     def _format_daily_html(self, data: Dict[str, Any]) -> str:
         """Format daily report as HTML."""
+        if data.get("lookback_days", 1) > 1:
+            new_trades_label = f"Trades Filed (last {data['lookback_days']} days):"
+        else:
+            new_trades_label = "New Trades:"
+
         html = f"""
         <html>
         <head>
@@ -307,7 +330,7 @@ class ReportGenerator:
             
             <div class="summary">
                 <h2>📊 Summary</h2>
-                <div class="stat"><strong>New Trades:</strong> {data['new_trades_count']}</div>
+                <div class="stat"><strong>{new_trades_label}</strong> {data['new_trades_count']}</div>
                 <div class="stat"><strong>Market Alerts:</strong> {data['market_alerts_count']}</div>
                 <div class="stat"><strong>Critical Alerts:</strong> {data['critical_alerts_count']}</div>
                 <div class="stat"><strong>Suspicious Trades:</strong> {data['suspicious_trades_count']}</div>