ci: allowlist known historical/fixture gitleaks findings

ci: fix gitleaks no-op ruleset (add useDefault = true)
ci: add local pre-commit gitleaks hook
2026-07-13 15:55:40 -04:00 · 2026-07-12 18:15:24 -05:00 · 2026-07-12 16:21:28 -05:00 · 2026-07-12 16:21:28 -05:00 · 2026-07-12 10:38:51 -05:00 · 2026-07-12 10:26:17 -05:00
23 changed files with 868 additions and 2393 deletions
@@ -45,7 +45,6 @@ logs/
 .DS_Store
 Thumbs.db
-# Docs (optional - include if you want them in container)
+# Docs (keep README.md — required by pyproject.toml / Docker build)
 docs/
 *.md
@@ -0,0 +1,84 @@
 ---
 # ci-sync: 2026-05-30T02:31:20Z
 # Homelab CI — Python lane (git-ci-01) + secret scan (git-ci-02)
 # Skip: @skipci in branch name or commit message
 name: CI
 on:
  push:
    branches: [master, main]
  pull_request:
    types: [opened, synchronize, reopened]
 jobs:
  skip-ci-check:
    runs-on: [homelab, self-hosted, linux]
    container:
      image: node:20-bookworm
    outputs:
      should-skip: ${{ steps.check.outputs.skip }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - id: check
        run: |
          SKIP=0
          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
          MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-$(git log -1 --pretty=%B 2>/dev/null || true)}"
          echo "$BRANCH" "$MSG" | grep -qi '@skipci' && SKIP=1
          echo "skip=$SKIP" >> $GITHUB_OUTPUT
  python-ci:
    needs: skip-ci-check
    if: needs.skip-ci-check.outputs.should-skip != '1'
    runs-on: [homelab, self-hosted, linux, python]
    container:
      # node image: actions/checkout@v4 needs Node; install python3 in-job
      image: node:20-bookworm
    steps:
      - uses: actions/checkout@v4
      - name: Install Python tooling
        run: |
          apt-get update -qq
          DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv
          python3 -m pip install --upgrade pip --break-system-packages
          if [ -f requirements.txt ]; then pip install -r requirements.txt --break-system-packages; fi
          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt --break-system-packages; fi
          pip install bandit pip-audit ruff --break-system-packages
      - name: Ruff lint
        run: ruff check . || true
      - name: Bandit (advisory)
        run: bandit -r . -q || true
      - name: pip-audit (advisory)
        run: pip-audit -r requirements.txt 2>/dev/null || pip-audit 2>/dev/null || true
      - name: Pytest
        run: |
          if [ -d tests ] || ls test_*.py *_test.py 2>/dev/null; then
            pip install pytest --break-system-packages
            pytest -q || true
          else
            echo "No tests found — skip"
          fi
  secret-scan:
    needs: skip-ci-check
    if: needs.skip-ci-check.outputs.should-skip != '1'
    runs-on: [homelab, self-hosted, linux, heavy]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Gitleaks
        run: |
          extra=""
          if [ -f .gitleaks.toml ]; then
            extra="--config /repo/.gitleaks.toml"
          fi
          docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \
            detect --source /repo --no-banner --redact ${extra}
@@ -3,16 +3,14 @@ name: CI
 on:
  push:
-    branches: [main, qa, dev]
+    branches: [main, master]
  pull_request:
    branches: [main, qa, dev]
 jobs:
  lint-and-test:
    runs-on: ubuntu-latest
-    container:
+    # No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images)
-      image: python:3.11-bullseye
+
    services:
      postgres:
        image: postgres:15
@@ -30,24 +28,20 @@ jobs:
      - name: Check out code
        uses: actions/checkout@v4
-      - name: Install system dependencies
+      - name: Set up Python venv
        run: |
-          apt-get update
+          python3 -m venv .venv
-          apt-get install -y postgresql-client
+          .venv/bin/pip install --upgrade pip
-
+          .venv/bin/pip install -e ".[dev]"
      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Run linters
        run: |
          echo "Running ruff..."
-          ruff check src/ tests/ || true
+          .venv/bin/ruff check src/ tests/ || true
          echo "Running black check..."
-          black --check src/ tests/ || true
+          .venv/bin/black --check src/ tests/ || true
          echo "Running mypy..."
-          mypy src/ --install-types --non-interactive || true
+          .venv/bin/mypy src/ --install-types --non-interactive || true
      - name: Run tests with coverage
        env:
@@ -58,62 +52,38 @@ jobs:
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }}
          FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }}
        run: |
-          pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
+          .venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
      - name: Test scripts
        env:
          DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test
        run: |
          echo "Testing database migrations..."
-          alembic upgrade head
+          .venv/bin/alembic upgrade head
          echo "Testing price loader..."
-          python scripts/fetch_sample_prices.py || true
+          .venv/bin/python scripts/fetch_sample_prices.py || true
  secret-scanning:
    runs-on: ubuntu-latest
    container:
      image: zricethezav/gitleaks:latest
    steps:
      - name: Install Node.js for checkout action
        run: |
          apk add --no-cache nodejs npm curl git
      - name: Check out code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Scan for secrets
        run: |
          echo "🔍 Scanning for exposed secrets..."
          gitleaks detect --source . --no-banner --redact --exit-code 0 || true
        continue-on-error: true
  security-scan:
    runs-on: ubuntu-latest
    container:
      image: python:3.11-bullseye
    steps:
      - name: Check out code
        uses: actions/checkout@v4
-      - name: Install dependencies
+      - name: Set up Python venv
        run: |
-          pip install --upgrade pip
+          python3 -m venv .venv
-          pip install safety bandit
+          .venv/bin/pip install --upgrade pip
          .venv/bin/pip install -e ".[dev]" safety bandit
      - name: Run safety check
        run: |
-          pip install -e .
+          .venv/bin/safety check --json || true
          echo "🔍 Checking for known vulnerabilities in dependencies..."
          safety check --json || true
        continue-on-error: true
      - name: Run bandit security scan
        run: |
-          echo "🔍 Running static security analysis..."
+          .venv/bin/bandit -r src/ -f json -o bandit-report.json || true
-          bandit -r src/ -f json -o bandit-report.json || true
+          .venv/bin/bandit -r src/ -f screen
          bandit -r src/ -f screen
        continue-on-error: true
  dependency-scan:
@@ -123,101 +93,13 @@ jobs:
    steps:
      - name: Install Node.js for checkout action
        run: |
-          apk add --no-cache nodejs npm curl git
+          apk add --no-cache nodejs npm curl
      - name: Check out code
        uses: actions/checkout@v4
      - name: Scan dependencies
-        run: |
+        run: trivy fs --scanners vuln --exit-code 0 .
          echo "🔍 Scanning dependencies for vulnerabilities..."
          trivy fs --scanners vuln --exit-code 0 .
  sast-scan:
    runs-on: ubuntu-latest
    container:
      image: ubuntu:22.04
    steps:
      - name: Install Node.js for checkout action
        run: |
          apt-get update && apt-get install -y curl git
          curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
          apt-get install -y nodejs
      - name: Check out code
        uses: actions/checkout@v4
      - name: Install Semgrep
        run: |
          apt-get update && apt-get install -y python3 python3-pip
          pip3 install semgrep
      - name: Run Semgrep scan
        run: |
          echo "🔍 Running SAST analysis with Semgrep..."
          semgrep --config=auto --error || true
        continue-on-error: true
  container-scan:
    runs-on: ubuntu-latest
    container:
      image: ubuntu:22.04
    steps:
      - name: Install Node.js for checkout action
        run: |
          apt-get update && apt-get install -y curl git
          curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
          apt-get install -y nodejs
      - name: Check out code
        uses: actions/checkout@v4
      - name: Install Trivy
        run: |
          set -e
          apt-get update && apt-get install -y wget curl tar
          # Use a fixed, known-good Trivy version
          TRIVY_VERSION="0.58.2"
          TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
          echo "Installing Trivy version: ${TRIVY_VERSION}"
          if ! wget --progress=bar:force "${TRIVY_URL}" -O /tmp/trivy.tar.gz 2>&1; then
            echo "❌ Failed to download Trivy"
            exit 1
          fi
          if [ ! -f /tmp/trivy.tar.gz ] || [ ! -s /tmp/trivy.tar.gz ]; then
            echo "❌ Downloaded Trivy archive is missing or empty"
            exit 1
          fi
          echo "Extracting Trivy..."
          if ! tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy; then
            echo "❌ Failed to extract Trivy"
            exit 1
          fi
          mv /tmp/trivy /usr/local/bin/trivy
          chmod +x /usr/local/bin/trivy
          trivy --version
      - name: Scan Dockerfile
        run: |
          if [ -f "Dockerfile" ]; then
            echo "🔍 Scanning Dockerfile for vulnerabilities..."
            trivy config Dockerfile || true
          else
            echo "No Dockerfile found, skipping scan"
          fi
        continue-on-error: true
      - name: Scan filesystem
        run: |
          echo "🔍 Scanning filesystem for vulnerabilities..."
          trivy fs --scanners vuln --severity HIGH,CRITICAL --format table . || true
        continue-on-error: true
  docker-build-test:
    runs-on: ubuntu-latest
@@ -225,25 +107,15 @@ jobs:
      - name: Check out code
        uses: actions/checkout@v4
-      - name: Set up Docker Buildx
+      - name: Build and test Docker image
        uses: docker/setup-buildx-action@v3
      - name: Build Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: false
          tags: pote:test
          cache-from: type=gha
          cache-to: type=gha,mode=max
      - name: Test Docker image
        run: |
-          docker run --rm pote:test python -c "import pote; print('✅ POTE import successful')"
+          # Plain docker build — buildx images are not visible to act_runner's docker run
          docker build -t pote:test .
          docker run --rm pote:test python -c "import pote; print('POTE import successful')"
  workflow-summary:
    runs-on: ubuntu-latest
-    needs: [lint-and-test, secret-scanning, security-scan, dependency-scan, sast-scan, container-scan, docker-build-test]
+    needs: [lint-and-test, security-scan, dependency-scan, docker-build-test]
    if: always()
    steps:
      - name: Generate workflow summary
@@ -255,21 +127,11 @@ jobs:
          echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true
          echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true
          echo "| 🧪 Lint & Test | ${{ needs.lint-and-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "| 🔐 Secret Scanning | ${{ needs.secret-scanning.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
-          echo "| 🔍 SAST Scan | ${{ needs.sast-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
+          echo "| 🐳 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "| 🐳 Container Scan | ${{ needs.container-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "| 🐋 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
          echo "" >> $GITHUB_STEP_SUMMARY || true
          echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true
          echo "" >> $GITHUB_STEP_SUMMARY || true
-          echo "All security and validation checks have completed." >> $GITHUB_STEP_SUMMARY || true
+          echo "All checks have completed. Review individual job logs for details." >> $GITHUB_STEP_SUMMARY || true
-          echo "" >> $GITHUB_STEP_SUMMARY || true
+
          echo "**Security Layers:**" >> $GITHUB_STEP_SUMMARY || true
          echo "- ✅ Secret scanning (Gitleaks)" >> $GITHUB_STEP_SUMMARY || true
          echo "- ✅ Dependency vulnerabilities (Safety + Trivy)" >> $GITHUB_STEP_SUMMARY || true
          echo "- ✅ Static security analysis (Bandit)" >> $GITHUB_STEP_SUMMARY || true
          echo "- ✅ SAST scanning (Semgrep)" >> $GITHUB_STEP_SUMMARY || true
          echo "- ✅ Container scanning (Trivy)" >> $GITHUB_STEP_SUMMARY || true
        continue-on-error: true
@@ -0,0 +1,30 @@
 # Homelab bootstrap — gitleaks allowlist (tests, examples, placeholders)
 #
 # IMPORTANT: `useDefault = true` is required — without it gitleaks loads ONLY
 # this file (title + allowlist) with ZERO detection rules, so it would never
 # flag a real secret. Fixed 2026-07 (security-hardening track); if you're
 # re-pushing this template to a repo that already had the old version, that
 # repo's secret scanning was a no-op until this lands.
 title = "homelab gitea bootstrap"
 [extend]
 useDefault = true
 [allowlist]
 description = "Test fixtures and example configs are not production secrets"
 paths = [
  '''(?i).*\.test\.(ts|tsx|js|jsx|py)$''',
  '''(?i).*\.spec\.(ts|tsx|js|jsx)$''',
  '''(?i).*/tests/.*''',
  '''(?i).*/__tests__/.*''',
  '''(?i).*\.example\.(yml|yaml|env|json|toml)$''',
  '''(?i).*vault\.example\.(yml|yaml)$''',
  '''(?i).*\.env\.example$''',
  '''(?i)CUSTOMIZATION_CHECKLIST\.md$''',
 ]
 regexes = [
  '''(?i)(invalid|fake|dummy|placeholder|example|changeme|change_me|not-a-real)''',
  '''(?i)sk-or-invalid''',
  '''(?i)msk-or-invalid''',
  '''(?i)your_ssh_private_key_here''',
 ]
@@ -1,439 +0,0 @@
 # CI Pipeline - Complete Security Suite
 **Enhanced CI/CD pipeline with comprehensive security scanning layers.**
 ---
 ## 🎯 What Changed
 ### ❌ Removed
 - **`ansible/` directory** - Moved to infrastructure repository (where it belongs)
 - **`ANSIBLE_INTEGRATION.md`** - Redundant with handoff docs
 ### ✅ Kept (Reference Documentation)
 - **`ANSIBLE_HANDOFF.md`** - Integration guide for your Ansible team
 - **`ANSIBLE_TECHNICAL_REFERENCE.md`** - Exact commands, paths, procedures
 - **`CUSTOMIZATION_CHECKLIST.md`** - Configuration reference
 - **`MOVE_ANSIBLE_TO_SEPARATE_REPO.md`** - Migration guide
 ### 🚀 Enhanced
 - **`.github/workflows/ci.yml`** - Added 5 new security scanning jobs
 ---
 ## 🔐 Security Layers
 ### 1. Secret Scanning (Gitleaks)
 **Tool:** [Gitleaks](https://github.com/gitleaks/gitleaks)  
 **Purpose:** Detect exposed secrets in code and git history
 **Scans for:**
 - API keys
 - Passwords
 - Tokens
 - Private keys
 - Database credentials
 **Features:**
 - Scans entire git history (`fetch-depth: 0`)
 - Redacted output (doesn't expose secrets in logs)
 - Continues on error (won't block CI)
 ```yaml
 container: zricethezav/gitleaks:latest
 command: gitleaks detect --source . --no-banner --redact --exit-code 0
 ```
 ---
 ### 2. Security Scan (Safety + Bandit)
 **Tools:** [Safety](https://pyup.io/safety/) + [Bandit](https://bandit.readthedocs.io/)
 #### Safety - Dependency Vulnerabilities
 **Purpose:** Check Python dependencies against CVE database
 **Scans for:**
 - Known vulnerabilities in packages
 - Outdated packages with security issues
 - CVE references
 ```bash
 pip install safety
 safety check --json
 ```
 #### Bandit - Static Security Analysis
 **Purpose:** Find common security issues in Python code
 **Scans for:**
 - SQL injection vulnerabilities
 - Hardcoded passwords
 - Use of `eval()`, `exec()`
 - Insecure temp file usage
 - Weak cryptography
 ```bash
 pip install bandit
 bandit -r src/ -f screen
 ```
 ---
 ### 3. Dependency Scan (Trivy)
 **Tool:** [Trivy](https://github.com/aquasecurity/trivy)  
 **Purpose:** Comprehensive vulnerability scanner
 **Scans:**
 - Python packages (from `pyproject.toml`)
 - System libraries
 - OS packages
 - CVE database
 ```yaml
 container: aquasec/trivy:latest
 command: trivy fs --scanners vuln --exit-code 0 .
 ```
 ---
 ### 4. SAST Scan (Semgrep)
 **Tool:** [Semgrep](https://semgrep.dev/)  
 **Purpose:** Static Application Security Testing
 **Scans for:**
 - Security anti-patterns
 - Code quality issues
 - Language-specific vulnerabilities
 - OWASP Top 10 issues
 **Features:**
 - Language-aware (understands Python syntax)
 - Pattern-based matching
 - Auto-config uses community rules
 ```bash
 pip install semgrep
 semgrep --config=auto --error
 ```
 ---
 ### 5. Container Scan (Trivy)
 **Tool:** Trivy (filesystem mode)  
 **Purpose:** Scan Docker configurations and filesystem
 **Scans:**
 - `Dockerfile` misconfigurations
 - Filesystem vulnerabilities
 - HIGH/CRITICAL severity issues
 **Features:**
 - Config scanning (Dockerfile best practices)
 - Filesystem scanning (all files)
 - Severity filtering
 ```bash
 trivy config Dockerfile
 trivy fs --scanners vuln --severity HIGH,CRITICAL --format table .
 ```
 ---
 ## 📊 CI Pipeline Jobs
 ### Complete Job List
 | Job | Purpose | Tool(s) | Blocking? |
 |-----|---------|---------|-----------|
 | **lint-and-test** | Code quality & tests | ruff, black, mypy, pytest | ✅ Yes |
 | **secret-scanning** | Exposed secrets | Gitleaks | ⚠️ No |
 | **security-scan** | Python security | Safety, Bandit | ⚠️ No |
 | **dependency-scan** | Dependency vulns | Trivy | ⚠️ No |
 | **sast-scan** | Static analysis | Semgrep | ⚠️ No |
 | **container-scan** | Container security | Trivy | ⚠️ No |
 | **docker-build-test** | Docker build | Docker | ✅ Yes |
 | **workflow-summary** | Status report | Native | ℹ️ Info |
 ### Blocking vs Non-Blocking
 **Blocking (will fail CI):**
 - `lint-and-test` - Code must pass tests
 - `docker-build-test` - Docker image must build
 **Non-Blocking (informational):**
 - All security scans use `continue-on-error: true`
 - Provides visibility without blocking development
 - Can be made blocking by removing `continue-on-error`
 ---
 ## 🎨 Workflow Summary
 After all jobs complete, a summary is generated:
 ```
 ## 🔍 CI Workflow Summary
 ### Job Results
 | Job | Status |
 |-----|--------|
 | 🧪 Lint & Test | success |
 | 🔐 Secret Scanning | success |
 | 🔒 Security Scan | success |
 | 📦 Dependency Scan | success |
 | 🔍 SAST Scan | success |
 | 🐳 Container Scan | success |
 | 🐋 Docker Build | success |
 ### 📊 Summary
 All security and validation checks have completed.
 **Security Layers:**
 - ✅ Secret scanning (Gitleaks)
 - ✅ Dependency vulnerabilities (Safety + Trivy)
 - ✅ Static security analysis (Bandit)
 - ✅ SAST scanning (Semgrep)
 - ✅ Container scanning (Trivy)
 ```
 ---
 ## 🔧 Configuration
 ### Gitea Secrets (Optional)
 None of the security scans require secrets, but you can configure:
 ```bash
 # Optional: For SonarQube integration (if you add it later)
 SONAR_HOST_URL=https://sonar.example.com
 SONAR_TOKEN=your_token_here
 ```
 ### Making Scans Blocking
 To make security scans fail the build, remove `continue-on-error: true`:
 ```yaml
 # Before (non-blocking):
 - name: Scan for secrets
  run: gitleaks detect --source . --no-banner --redact --exit-code 0
  continue-on-error: true
 # After (blocking):
 - name: Scan for secrets
  run: gitleaks detect --source . --no-banner --redact --exit-code 1
  # Removed continue-on-error
 ```
 ---
 ## 📈 Comparison with Your Ansible Pipeline
 ### Features from Your Pipeline
 | Feature | Your Ansible Pipeline | POTE Pipeline | Status |
 |---------|----------------------|---------------|--------|
 | Markdown linting | ✅ npm run test:markdown | ❌ N/A | Not needed (Python project) |
 | Ansible validation | ✅ ansible-lint | ❌ Removed | Moved to infrastructure repo |
 | Secret scanning | ✅ Gitleaks | ✅ Gitleaks | ✅ Implemented |
 | Dependency scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
 | SAST scan | ✅ Semgrep | ✅ Semgrep | ✅ Implemented |
 | License check | ✅ license-checker (npm) | ❌ N/A | Not needed (MIT license) |
 | Vault check | ✅ Ansible vault | ❌ Removed | No vault files in app repo |
 | Playbook test | ✅ ansible-playbook | ❌ Removed | No playbooks in app repo |
 | Container scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
 | SonarQube | ✅ sonar-scanner | ❌ Not added | Can add if needed |
 ### What's Different
 **Removed (Ansible-specific):**
 - Ansible linting
 - Vault validation
 - Playbook syntax checks
 - Markdown linting (not applicable)
 **Added (Python-specific):**
 - Python linting (ruff, black, mypy)
 - pytest with coverage
 - Safety (Python dependency CVE check)
 - Bandit (Python security linter)
 **Kept (Universal):**
 - Secret scanning (Gitleaks)
 - Dependency scanning (Trivy)
 - SAST scanning (Semgrep)
 - Container scanning (Trivy)
 ---
 ## 🚀 Usage
 ### Triggers
 The pipeline runs on:
 ```yaml
 on:
  push:
    branches: [main, qa, dev]
  pull_request:
    branches: [main, qa, dev]
 ```
 ### Manual Trigger
 To trigger manually (if you add `workflow_dispatch`):
 ```yaml
 on:
  workflow_dispatch:  # Add this
  push:
    branches: [main, qa, dev]
  pull_request:
    branches: [main, qa, dev]
 ```
 Then trigger via Gitea UI: Actions → CI → Run workflow
 ---
 ## 📊 Viewing Results
 ### In Gitea
 1. **Navigate to:** Repository → Actions → CI workflow
 2. **Click on:** Latest run
 3. **View:** Individual job logs
 4. **Summary:** Scroll to bottom for workflow summary
 ### Locally
 Run the same checks locally before pushing:
 ```bash
 # Linting
 ruff check src/ tests/
 black --check src/ tests/
 mypy src/
 # Tests
 pytest tests/ -v --cov=src/pote
 # Security scans (if tools installed)
 gitleaks detect --source . --no-banner
 safety check
 bandit -r src/
 semgrep --config=auto src/
 trivy fs .
 ```
 ---
 ## 🔄 Future Enhancements
 ### Optional Additions
 1. **SonarQube Integration**
   - Code quality metrics
   - Technical debt tracking
   - Requires SonarQube server
 2. **License Checking**
   - Scan Python dependencies for licenses
   - Tool: `pip-licenses`
 3. **Performance Testing**
   - Benchmark critical functions
   - Tool: `pytest-benchmark`
 4. **Code Coverage Gates**
   - Fail if coverage drops below threshold
   - Already have coverage reporting
 5. **Dependency Updates**
   - Auto-create PRs for dependency updates
   - Tool: Dependabot or Renovate
 ---
 ## 🆘 Troubleshooting
 ### Job Failing: secret-scanning
 **Issue:** Gitleaks found exposed secrets
 **Solution:**
 1. Review the scan output (redacted)
 2. Remove secrets from code
 3. Use `.env` files (already in `.gitignore`)
 4. Rotate exposed credentials
 ### Job Failing: security-scan
 **Issue:** Safety found vulnerable dependencies
 **Solution:**
 1. Review `safety check` output
 2. Update vulnerable packages: `pip install --upgrade <package>`
 3. Update `pyproject.toml` with new versions
 ### Job Failing: sast-scan
 **Issue:** Semgrep found security issues
 **Solution:**
 1. Review Semgrep output
 2. Fix identified issues
 3. Add `# nosemgrep` comment if false positive
 ### Job Failing: container-scan
 **Issue:** Trivy found HIGH/CRITICAL vulnerabilities
 **Solution:**
 1. Review Trivy output
 2. Update base image in `Dockerfile`
 3. Update system packages
 ---
 ## 📝 Best Practices
 ### 1. Review Security Scan Results
 - Don't ignore security warnings
 - Investigate all HIGH/CRITICAL findings
 - Keep dependencies up to date
 ### 2. Use Secrets Management
 - Never commit secrets
 - Use Gitea secrets for CI/CD
 - Use `.env` files locally (in `.gitignore`)
 ### 3. Keep Tools Updated
 - Security tools are frequently updated
 - Pin versions for stability
 - Update quarterly
 ### 4. Make Critical Scans Blocking
 - Consider making secret scanning blocking
 - Consider making HIGH/CRITICAL vulnerability scans blocking
 ---
 ## 📞 Support
 - **CI Pipeline Issues:** Check `.github/workflows/ci.yml`
 - **Security Tool Docs:**
  - [Gitleaks](https://github.com/gitleaks/gitleaks)
  - [Safety](https://pyup.io/safety/)
  - [Bandit](https://bandit.readthedocs.io/)
  - [Trivy](https://github.com/aquasecurity/trivy)
  - [Semgrep](https://semgrep.dev/)
 ---
 **Last Updated:** December 2025  
 **Pipeline Version:** 2.0 (Enhanced Security Suite)  
 **Total Security Layers:** 5
@@ -1,311 +0,0 @@
 # POTE Customization Checklist
 **Everything you need to change from generic defaults to your specific deployment.**
 ---
 ## 🔴 CRITICAL - Must Change (Security & Functionality)
 ### 1. Email Configuration (`.env` file)
 ```bash
 # Current generic values:
 SMTP_HOST=mail.levkin.ca          # ✅ Already yours
 SMTP_PORT=587                      # ✅ OK (standard)
 SMTP_USER=test@levkin.ca          # ✅ Already yours
 SMTP_PASSWORD=your_password_here   # 🔴 CHANGE THIS
 FROM_EMAIL=test@levkin.ca         # ✅ Already yours
 REPORT_RECIPIENTS=admin@localhost  # 🔴 CHANGE THIS to your email
 ```
 **Action:** Update `SMTP_PASSWORD` and `REPORT_RECIPIENTS` in `.env`
 ### 2. Database Password (`.env` file)
 ```bash
 # Current generic value:
 DATABASE_URL=postgresql://poteuser:changeme123@localhost:5432/potedb
 # 🔴 CHANGE "changeme123" to a strong password
 ```
 **Action:** Choose a strong password and update in:
 - `.env` file
 - PostgreSQL (if already created): `ALTER USER poteuser PASSWORD 'your_new_password';`
 ### 3. Git Repository (Ansible: `ansible/group_vars/all.yml`)
 ```yaml
 # Current value:
 pote_git_repo: "gitea@10.0.30.169:ilia/POTE.git"
 # 🔴 This is YOUR Gitea repo, but verify:
 # - IP address: 10.0.30.169 (is this correct?)
 # - Username: ilia (is this correct?)
 # - Repo name: POTE (is this correct?)
 ```
 **Action:** Verify or update the Git repo URL
 ### 4. SSH Keys (Ansible: `ansible/vault.example.yml`)
 ```yaml
 # 🔴 MUST CREATE vault.yml with your actual keys:
 vault_git_ssh_key: |
  -----BEGIN OPENSSH PRIVATE KEY-----
  your_ssh_private_key_here  # 🔴 ADD YOUR KEY
  -----END OPENSSH PRIVATE KEY-----
 vault_ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC..."  # 🔴 ADD YOUR KEY
 ```
 **Action:** 
 1. Copy `ansible/vault.example.yml` → `ansible/group_vars/all/vault.yml`
 2. Add your SSH keys
 3. Encrypt: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
 ### 5. Server IP Addresses (Ansible: `ansible/inventory.example.yml`)
 ```yaml
 # Current values:
 development:
  hosts:
    pote-dev:
      ansible_host: 10.0.10.100  # 🔴 CHANGE to your dev server IP
 staging:
  hosts:
    pote-qa:
      ansible_host: 10.0.10.101  # 🔴 CHANGE to your QA server IP
 production:
  hosts:
    pote-prod:
      ansible_host: 10.0.10.95   # 🔴 CHANGE to your prod server IP (or keep if correct)
 ```
 **Action:** Update all IP addresses to match your Proxmox LXC containers
 ---
 ## 🟡 IMPORTANT - Should Change (Gitea Secrets)
 ### 6. Gitea Secrets (for CI/CD pipelines)
 **Location:** Gitea Web UI → Repository Settings → Secrets
 ```bash
 # Required secrets:
 DB_PASSWORD=changeme123              # 🟡 CHANGE to match your DB password
 SMTP_PASSWORD=your_password_here     # 🟡 CHANGE to your email password
 SMTP_HOST=mail.levkin.ca            # ✅ Already yours
 SMTP_USER=test@levkin.ca            # ✅ Already yours
 FROM_EMAIL=test@levkin.ca           # ✅ Already yours
 # For deployment workflow (if using):
 PROXMOX_SSH_KEY=<your_private_key>   # 🟡 ADD your SSH private key
 PROXMOX_HOST=10.0.10.95             # 🟡 CHANGE to your server IP
 PROXMOX_USER=poteapp                # 🟡 CHANGE if using different user
 ```
 **Action:** Add/update secrets in Gitea:
 1. Go to `https://git.levkin.ca/ilia/POTE/settings/secrets`
 2. Add each secret listed above
 ---
 ## 🟢 OPTIONAL - Customize for Your Needs
 ### 7. Email Recipients (Multiple locations)
 **`.env` file:**
 ```bash
 REPORT_RECIPIENTS=admin@localhost  # 🟢 Change to your email(s), comma-separated
 ```
 **`scripts/automated_daily_run.sh`:**
 ```bash
 REPORT_RECIPIENTS="${REPORT_RECIPIENTS:-admin@localhost}"  # 🟢 Change default
 ```
 **`scripts/setup_cron.sh`:**
 - Will prompt you interactively for email address
 **Action:** Update to your preferred email(s) for reports
 ### 8. Market Monitoring Tickers (`.env` and Ansible)
 **`.env` file:**
 ```bash
 MARKET_MONITOR_TICKERS=NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX
 # 🟢 Customize this list based on what Congress trades most
 ```
 **`ansible/group_vars/all.yml`:**
 ```yaml
 market_tickers: "NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX"
 # 🟢 Should match .env
 ```
 **Action:** Research most-traded congressional stocks and update list
 ### 9. Alert Severity Threshold (`.env` and Ansible)
 **`.env` file:**
 ```bash
 ALERT_MIN_SEVERITY=5  # 🟢 1-10, lower = more sensitive
 ```
 **`ansible/group_vars/all.yml`:**
 ```yaml
 alert_severity: 5  # 🟢 Should match .env
 ```
 **Action:** Adjust based on how many alerts you want (5 is moderate)
 ### 10. Cron Schedule Times (Ansible)
 **`ansible/group_vars/development.yml`, `staging.yml`, `production.yml`:**
 ```yaml
 pote_daily_report_time: "0 6"      # 🟢 6:00 AM - change to your preference
 pote_weekly_report_time: "0 8"     # 🟢 8:00 AM Sunday - change to your preference
 pote_health_check_time: "*/30 * * * *"  # 🟢 Every 30 min - change to your preference
 ```
 **Action:** Adjust times based on your timezone and preferences
 ### 11. Log Level (`.env` and Ansible)
 **`.env` file:**
 ```bash
 LOG_LEVEL=INFO  # 🟢 DEBUG, INFO, WARNING, ERROR
 ```
 **`ansible/group_vars/all.yml`:**
 ```yaml
 log_level: "INFO"  # 🟢 Should match .env
 ```
 **Action:** Use `DEBUG` for development, `INFO` for production
 ### 12. Application User (Ansible)
 **`ansible/group_vars/all.yml`:**
 ```yaml
 appuser_name: "poteapp"  # 🟢 Change if you want a different username
 ```
 **`scripts/proxmox_setup.sh`:**
 ```bash
 APP_USER="poteapp"  # 🟢 Should match Ansible
 ```
 **Action:** Keep as `poteapp` unless you have a specific naming convention
 ### 13. Database Names (Ansible - per environment)
 **`ansible/group_vars/development.yml`:**
 ```yaml
 db_name: "potedb_dev"  # 🟢 OK as-is
 ```
 **`ansible/group_vars/staging.yml`:**
 ```yaml
 db_name: "potedb_qa"   # 🟢 OK as-is
 ```
 **`ansible/group_vars/production.yml`:**
 ```yaml
 db_name: "potedb"      # 🟢 OK as-is
 ```
 **Action:** Keep defaults unless you have a specific naming convention
 ### 14. Backup Retention (Ansible)
 **`ansible/roles/pote/defaults/main.yml`:**
 ```yaml
 pote_backup_retention_days: 90  # 🟢 Adjust based on disk space
 ```
 **Action:** Increase for production (180+ days), decrease for dev (30 days)
 ### 15. API Keys (`.env` - if you get paid APIs)
 **`.env` file:**
 ```bash
 QUIVERQUANT_API_KEY=              # 🟢 Optional paid service
 FMP_API_KEY=                      # 🟢 Optional paid service
 ```
 **Action:** Add keys if you subscribe to these services (not required)
 ---
 ## 📋 Summary: Quick Action List
 ### Immediate (Before First Deployment)
 1. ✅ Update `.env`: `SMTP_PASSWORD`, `REPORT_RECIPIENTS`, `DATABASE_URL` password
 2. ✅ Create `ansible/group_vars/all/vault.yml` with your SSH keys
 3. ✅ Encrypt vault: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
 4. ✅ Update `ansible/inventory.yml` with your server IPs
 5. ✅ Add Gitea secrets: `DB_PASSWORD`, `SMTP_PASSWORD`
 ### Before Production Use
 6. ✅ Verify Git repo URL in `ansible/group_vars/all.yml`
 7. ✅ Customize `MARKET_MONITOR_TICKERS` based on research
 8. ✅ Adjust cron times for your timezone
 9. ✅ Set appropriate log levels per environment
 ### Optional Enhancements
 10. ⭐ Add paid API keys if you subscribe
 11. ⭐ Adjust alert sensitivity based on testing
 12. ⭐ Customize backup retention per environment
 ---
 ## 🔍 How to Find These Files
 ```bash
 # Configuration files:
 .env                                    # Main config (not in git)
 src/pote/config.py                      # Python config loader
 # Ansible files:
 ansible/inventory.yml                   # Server IPs (copy from .example.yml)
 ansible/group_vars/all.yml             # Common variables
 ansible/group_vars/all/vault.yml       # Secrets (create from vault.example.yml)
 ansible/group_vars/development.yml     # Dev environment
 ansible/group_vars/staging.yml         # QA environment
 ansible/group_vars/production.yml      # Prod environment
 ansible/roles/pote/defaults/main.yml   # All default variables
 # Scripts:
 scripts/automated_daily_run.sh         # Daily automation
 scripts/automated_weekly_run.sh        # Weekly automation
 scripts/setup_cron.sh                  # Cron setup
 scripts/proxmox_setup.sh               # Initial server setup
 # CI/CD:
 .github/workflows/ci.yml               # CI pipeline
 .github/workflows/deploy.yml           # Deployment pipeline
 ```
 ---
 ## 🚨 Security Reminders
 1. **NEVER commit `.env` to git** - it's in `.gitignore`
 2. **NEVER commit unencrypted `vault.yml`** - always use `ansible-vault encrypt`
 3. **NEVER put real passwords in example files** - use placeholders
 4. **ALWAYS use strong passwords** - minimum 16 characters, mixed case, numbers, symbols
 5. **ALWAYS use Gitea secrets** for CI/CD - never hardcode in workflow files
 ---
 ## 📞 Need Help?
 - **Ansible Vault:** `ansible-vault --help`
 - **Gitea Secrets:** Repository Settings → Secrets → Actions
 - **Environment Variables:** See `src/pote/config.py` for all available settings
 - **Testing Config:** Run `python scripts/health_check.py` after changes
 ---
 **Last Updated:** December 2025
@@ -1,18 +1,18 @@
-# Email Setup for levkin.ca
+# Email Setup for levkine.ca (Mailcow)
-Your POTE system is configured to use `test@levkin.ca` for sending reports.
+Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`.
 ## ✅ Configuration Done
 The `.env` file has been created with these settings:
 ```env
-SMTP_HOST=mail.levkin.ca
+SMTP_HOST=10.0.10.132
 SMTP_PORT=587
-SMTP_USER=test@levkin.ca
+SMTP_USER=alerts@levkine.ca
 SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE
-FROM_EMAIL=test@levkin.ca
+FROM_EMAIL=alerts@levkine.ca
-REPORT_RECIPIENTS=test@levkin.ca
+REPORT_RECIPIENTS=idobkin@gmail.com
 ```
 ## 🔑 Next Steps
@@ -30,6 +30,8 @@ POTE tracks stock trading activity of government officials (starting with U.S. C
 **📧 Want automated reports?** See **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** for email reporting setup!
 **🏠 Homelab deploy (LXC 236)?** See **[docs/HANDOFF-2026-05-27.md](docs/HANDOFF-2026-05-27.md)** for ops handoff and next steps.
 ### Local Development
 ```bash
 # Install
@@ -0,0 +1,324 @@
 # POTE Testing Status Report
 **Date:** December 15, 2025  
 **Status:** ✅ All Systems Operational - Ready for Deployment
 ---
 ## 🎯 Test Suite Summary
 ### **55 Tests - All Passing ✅**
 ```
 Platform: Python 3.13.5, pytest-9.0.2
 Test Duration: ~1.8 seconds
 Coverage: ~85% overall
 ```
 ### Test Breakdown by Module:
 | Module | Tests | Status | Coverage |
 |--------|-------|--------|----------|
 | **Analytics** | 18 tests | ✅ PASS | 80% |
 | **Models** | 7 tests | ✅ PASS | 90% |
 | **Ingestion** | 14 tests | ✅ PASS | 85% |
 | **Price Loader** | 8 tests | ✅ PASS | 90% |
 | **Security Enricher** | 8 tests | ✅ PASS | 85% |
 ---
 ## 📊 What's Been Tested?
 ### ✅ Core Database Operations
 - [x] Creating and querying Officials
 - [x] Creating and querying Securities  
 - [x] Creating and querying Trades
 - [x] Price data storage and retrieval
 - [x] Unique constraints and relationships
 - [x] Database migrations (Alembic)
 ### ✅ Data Ingestion
 - [x] House Stock Watcher client (with fixtures)
 - [x] Trade loading from JSON
 - [x] Security enrichment from yfinance
 - [x] Price data fetching and storage
 - [x] Idempotent operations (no duplicates)
 - [x] Error handling for missing/invalid data
 ### ✅ Analytics Engine
 - [x] Return calculations (buy trades)
 - [x] Return calculations (sell trades)
 - [x] Multiple time windows (30/60/90/180 days)
 - [x] Benchmark comparisons (SPY, QQQ, etc.)
 - [x] Abnormal returns (alpha calculations)
 - [x] Official performance summaries
 - [x] Sector-level analysis
 - [x] Disclosure timing analysis
 - [x] Top performer rankings
 - [x] System-wide statistics
 ### ✅ Edge Cases
 - [x] Missing price data handling
 - [x] Trades with no exit price yet
 - [x] Sell trades (inverted returns)
 - [x] Disclosure lags
 - [x] Duplicate prevention
 - [x] Invalid date ranges
 - [x] Empty result sets
 ---
 ## 🧪 Test Types
 ### 1. Unit Tests (Fast, Isolated)
 **Location:** `tests/test_*.py` (excluding integration)  
 **Purpose:** Test individual functions and classes  
 **Database:** In-memory SQLite (fresh for each test)  
 **Speed:** ~0.5 seconds
 **Examples:**
 - `test_parse_amount_range()` - Parse trade amounts
 - `test_normalize_transaction_type()` - Trade type normalization
 - `test_get_or_create_security()` - Security deduplication
 ### 2. Integration Tests (Realistic Scenarios)
 **Location:** `tests/test_analytics_integration.py`  
 **Purpose:** Test complete workflows with synthetic data  
 **Database:** In-memory SQLite with realistic price data  
 **Speed:** ~0.7 seconds
 **Examples:**
 - `test_return_calculation_with_real_data()` - Full return calc pipeline
 - `test_benchmark_comparison_with_real_data()` - Alpha calculations
 - `test_official_performance_summary()` - Aggregated metrics
 **Scenarios Tested:**
 - Nancy Pelosi buys NVDA early (strong returns)
 - Tommy Tuberville buys NVDA later (good but less alpha)
 - 120 days of synthetic price data (realistic trends)
 - SPY benchmark comparison
 - Multiple time window analysis
 ---
 ## 🔧 How to Run Tests Locally
 ### Quick Test
 ```bash
 cd /home/user/Documents/code/pote
 source venv/bin/activate
 pytest -v
 ```
 ### With Coverage Report
 ```bash
 pytest --cov=src/pote --cov-report=html --cov-report=term
 # View: firefox htmlcov/index.html
 ```
 ### Specific Test Modules
 ```bash
 # Just analytics
 pytest tests/test_analytics.py -v
 # Just integration tests
 pytest tests/test_analytics_integration.py -v
 # Specific test
 pytest tests/test_analytics.py::test_return_calculator_basic -v
 ```
 ### Watch Mode (Re-run on changes)
 ```bash
 pytest-watch
 # or
 ptw
 ```
 ---
 ## 🚨 Known Limitations
 ### 1. External API Dependency
 **Issue:** House Stock Watcher API is currently DOWN  
 **Impact:** Can't fetch live congressional trades automatically  
 **Workaround:** 
 - Use fixtures (`scripts/ingest_from_fixtures.py`)
 - Manual CSV import (`scripts/scrape_alternative_sources.py`)
 - Manual entry (`scripts/add_custom_trades.py`)
 ### 2. Market Data Limits
 **Issue:** yfinance has rate limits and occasional failures  
 **Impact:** Bulk price fetching may be slow  
 **Workaround:**
 - Fetch in batches
 - Add retry logic (already implemented)
 - Use caching (already implemented)
 ### 3. No Live Trading API
 **Issue:** We only use public disclosure data (inherent lag)  
 **Impact:** Trades are 30-45 days delayed by law  
 **This is expected:** POTE is for research, not real-time trading
 ---
 ## 📈 Performance Benchmarks
 ### Test Execution Time
 - **Full suite:** 1.8 seconds
 - **Unit tests only:** 0.5 seconds  
 - **Integration tests:** 0.7 seconds
 - **Parallel execution:** ~1.0 second (with pytest-xdist)
 ### Database Operations
 - **Create official:** < 1ms
 - **Create trade:** < 1ms
 - **Fetch prices (100 days):** ~50ms (in-memory)
 - **Calculate returns:** ~10ms per trade
 - **Aggregate metrics:** ~50ms for 100 trades
 ---
 ## 🎯 Pre-Deployment Checklist
 ### Before Deploying to Proxmox:
 - [x] All tests passing locally
 - [x] No linter errors (`make lint`)
 - [x] Database migrations work (`alembic upgrade head`)
 - [x] Scripts are executable and work
 - [x] Environment variables documented
 - [x] Sample data available for testing
 - [x] Documentation up to date
 ### On Proxmox Container:
 ```bash
 # 1. Pull latest code
 cd ~/pote
 git pull
 # 2. Update dependencies
 pip install -e .
 # 3. Run tests
 pytest -v
 # 4. Run migrations
 alembic upgrade head
 # 5. Verify system
 python ~/status.sh
 # 6. Test a script
 python scripts/enrich_securities.py
 ```
 ---
 ## 🔄 Continuous Testing
 ### Git Pre-Commit Hook (Optional)
 ```bash
 #!/bin/bash
 # .git/hooks/pre-commit
 pytest --tb=short
 if [ $? -ne 0 ]; then
    echo "Tests failed. Commit aborted."
    exit 1
 fi
 ```
 ### CI/CD Integration (Future)
 When you set up GitHub Actions or GitLab CI:
 ```yaml
 # .github/workflows/test.yml
 name: Tests
 on: [push, pull_request]
 jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
      - run: pip install -e .
      - run: pytest -v --cov
 ```
 ---
 ## 📝 Test Maintenance
 ### Adding New Tests
 **When to add tests:**
 - Adding new features
 - Fixing bugs (write test that fails, then fix)
 - Before refactoring (ensure tests pass before & after)
 **Where to add tests:**
 - Unit tests: `tests/test_<module>.py`
 - Integration tests: `tests/test_<feature>_integration.py`
 **Example:**
 ```python
 def test_new_feature(test_db_session):
    """Test description."""
    session = test_db_session
    # Arrange
    # Act
    # Assert
 ```
 ### Updating Fixtures
 Fixtures are in `tests/conftest.py`:
 - `test_db_session` - Fresh database
 - `sample_official` - Test official
 - `sample_security` - Test security (AAPL)
 - `sample_trade` - Test trade
 - `sample_price` - Test price record
 ---
 ## 🎉 Summary
 ### Current Status: **PRODUCTION READY** ✅
 **What Works:**
 - ✅ All 55 tests passing
 - ✅ Full analytics pipeline functional
 - ✅ Database operations solid
 - ✅ Data ingestion from multiple sources
 - ✅ Price fetching from yfinance
 - ✅ Security enrichment
 - ✅ Return calculations
 - ✅ Benchmark comparisons
 - ✅ Performance metrics
 - ✅ CLI scripts operational
 **What's Missing:**
 - ❌ Live congressional trade API (external issue - House Stock Watcher down)
  - **Workaround:** Manual import, CSV, or alternative APIs available
 **Next Steps:**
 1. ✅ Tests are complete
 2. ✅ Code is ready
 3. ➡️ **Deploy to Proxmox** (or continue with Phase 2 features)
 4. ➡️ Add more data sources
 5. ➡️ Build dashboard (Phase 3)
 ---
 ## 📞 Need Help?
 See:
 - `LOCAL_TEST_GUIDE.md` - Detailed local testing instructions
 - `QUICKSTART.md` - Usage guide for deployed system
 - `docs/09_data_updates.md` - How to add/update data
 - `README.md` - Project overview
 **Questions about testing?**  
 All tests are documented with docstrings - read the test files!
@@ -1,681 +0,0 @@
 # Branch Strategy & Multi-Environment Deployment
 ## Overview
 This guide covers setting up a proper Git branching strategy with protected branches for Dev, QA, and Production environments, integrated with your Ansible auto-deployment system.
 ---
 ## 🌳 Branch Strategy
 ### Recommended Branch Structure
 ```
 main (production)
 ├── qa (quality assurance/staging)
 └── dev (development)
 ```
 **Alternative naming:**
 ```
 prod (production)
 ├── staging (QA)
 └── develop (dev)
 ```
 ---
 ## 🔒 Branch Protection Rules
 ### In Gitea: Repository Settings → Branches
 #### 1. `main` (Production) - MOST PROTECTED
 **Protection Rules:**
 - ✅ **Require pull request reviews** (at least 1 approval)
 - ✅ **Require status checks to pass** (CI must pass)
 - ✅ **Restrict who can push** (only maintainers)
 - ✅ **Require signed commits** (optional but recommended)
 - ✅ **Block force pushes**
 - ✅ **Block deletions**
 - ✅ **Require linear history** (no merge commits)
 **Merge Strategy:**
 - Only merge from `qa` branch
 - Require successful QA testing
 - Tag releases: `v1.0.0`, `v1.1.0`, etc.
 #### 2. `qa` (Staging) - MODERATELY PROTECTED
 **Protection Rules:**
 - ✅ **Require pull request reviews** (at least 1 approval)
 - ✅ **Require status checks to pass** (CI must pass)
 - ✅ **Block force pushes**
 - ✅ **Block deletions**
 **Merge Strategy:**
 - Merge from `dev` branch
 - Run full test suite
 - Manual QA testing required
 #### 3. `dev` (Development) - LIGHTLY PROTECTED
 **Protection Rules:**
 - ✅ **Require status checks to pass** (CI must pass)
 - ✅ **Block force pushes** (optional)
 - ⚠️ Allow direct commits (for rapid development)
 **Merge Strategy:**
 - Feature branches merge here
 - Continuous integration testing
 - Auto-deploy to dev environment
 ---
 ## 📋 Gitea Branch Protection Setup
 ### Step-by-Step Configuration
 #### 1. Create Branches
 ```bash
 cd /home/user/Documents/code/pote
 # Create dev branch from main
 git checkout -b dev
 git push origin dev
 # Create qa branch from main
 git checkout -b qa
 git push origin qa
 # Back to main
 git checkout main
 ```
 #### 2. Configure in Gitea UI
 ```
 1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
 2. Click "Add New Rule"
 For MAIN branch:
 - Branch name pattern: main
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require 1 approval
 - ✅ Require status checks
 - ✅ Block force push
 - ✅ Block deletion
 - Whitelist: (leave empty or add specific users)
 For QA branch:
 - Branch name pattern: qa
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require status checks
 - ✅ Block force push
 - ✅ Block deletion
 For DEV branch:
 - Branch name pattern: dev
 - ✅ Require status checks
 - ⚠️ Allow direct push (for development)
 ```
 ---
 ## 🚀 Deployment Workflow Integration
 ### Update Workflows for Multi-Environment
 #### 1. Update CI Workflow for All Branches
 **File:** `.github/workflows/ci.yml`
 ```yaml
 name: CI
 on:
  push:
    branches: [main, qa, dev]
  pull_request:
    branches: [main, qa, dev]
 jobs:
  lint-and-test:
    runs-on: ubuntu-latest
    # ... existing CI jobs ...
 ```
 #### 2. Create Environment-Specific Deployment Workflows
 **File:** `.github/workflows/deploy-dev.yml`
 ```yaml
 name: Deploy to Dev
 on:
  push:
    branches: [dev]
  workflow_dispatch:
 jobs:
  deploy-dev:
    runs-on: ubuntu-latest
    environment: development
    steps:
      - name: Check out code
        uses: actions/checkout@v4
      - name: Deploy to Dev Server
        env:
          DEV_HOST: ${{ secrets.DEV_HOST }}
          DEV_USER: ${{ secrets.DEV_USER }}
          DEV_SSH_KEY: ${{ secrets.DEV_SSH_KEY }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_DEV }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD_DEV }}
        run: |
          # Setup SSH
          mkdir -p ~/.ssh
          echo "$DEV_SSH_KEY" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan -H $DEV_HOST >> ~/.ssh/known_hosts
          # Deploy
          ssh ${DEV_USER}@${DEV_HOST} << 'ENDSSH'
            cd ~/pote-dev
            git fetch origin
            git checkout dev
            git pull origin dev
            source venv/bin/activate
            pip install -e .
            alembic upgrade head
          ENDSSH
 ```
 **File:** `.github/workflows/deploy-qa.yml`
 ```yaml
 name: Deploy to QA
 on:
  push:
    branches: [qa]
  workflow_dispatch:
 jobs:
  deploy-qa:
    runs-on: ubuntu-latest
    environment: staging
    steps:
      - name: Check out code
        uses: actions/checkout@v4
      - name: Deploy to QA Server
        env:
          QA_HOST: ${{ secrets.QA_HOST }}
          QA_USER: ${{ secrets.QA_USER }}
          QA_SSH_KEY: ${{ secrets.QA_SSH_KEY }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_QA }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD_QA }}
        run: |
          # Setup SSH
          mkdir -p ~/.ssh
          echo "$QA_SSH_KEY" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan -H $QA_HOST >> ~/.ssh/known_hosts
          # Deploy
          ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
            cd ~/pote-qa
            git fetch origin
            git checkout qa
            git pull origin qa
            source venv/bin/activate
            pip install -e .
            alembic upgrade head
          ENDSSH
      - name: Run Smoke Tests
        run: |
          ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
            cd ~/pote-qa
            source venv/bin/activate
            python scripts/health_check.py
          ENDSSH
 ```
 **File:** `.github/workflows/deploy-prod.yml`
 ```yaml
 name: Deploy to Production
 on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      confirm:
        description: 'Type "DEPLOY" to confirm production deployment'
        required: true
 jobs:
  deploy-prod:
    runs-on: ubuntu-latest
    environment: production
    if: github.event.inputs.confirm == 'DEPLOY' || github.event_name == 'push'
    steps:
      - name: Check out code
        uses: actions/checkout@v4
      - name: Create Release Tag
        run: |
          git tag -a "v$(date +%Y%m%d-%H%M%S)" -m "Production release"
          git push origin --tags
      - name: Deploy to Production
        env:
          PROD_HOST: ${{ secrets.PROXMOX_HOST }}
          PROD_USER: ${{ secrets.PROXMOX_USER }}
          PROD_SSH_KEY: ${{ secrets.PROXMOX_SSH_KEY }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
        run: |
          # Setup SSH
          mkdir -p ~/.ssh
          echo "$PROD_SSH_KEY" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan -H $PROD_HOST >> ~/.ssh/known_hosts
          # Backup current production
          ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
            cd ~/pote
            git tag "backup-$(date +%Y%m%d-%H%M%S)"
          ENDSSH
          # Deploy
          ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
            cd ~/pote
            git fetch origin
            git checkout main
            git pull origin main
            source venv/bin/activate
            pip install -e .
            alembic upgrade head
          ENDSSH
      - name: Health Check
        run: |
          ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
            cd ~/pote
            source venv/bin/activate
            python scripts/health_check.py
          ENDSSH
      - name: Rollback on Failure
        if: failure()
        run: |
          echo "❌ Deployment failed, rolling back..."
          ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
            cd ~/pote
            latest_backup=$(git tag -l "backup-*" | sort -r | head -1)
            git checkout "$latest_backup"
            source venv/bin/activate
            alembic upgrade head
          ENDSSH
 ```
 ---
 ## 🔐 Gitea Secrets for Multi-Environment
 ### Organize Secrets by Environment
 #### Development Secrets
 ```
 DEV_HOST=10.0.10.100
 DEV_USER=poteapp
 DEV_SSH_KEY=(dev SSH key)
 SMTP_PASSWORD_DEV=(dev mail password)
 DB_PASSWORD_DEV=dev_password_123
 ```
 #### QA/Staging Secrets
 ```
 QA_HOST=10.0.10.101
 QA_USER=poteapp
 QA_SSH_KEY=(qa SSH key)
 SMTP_PASSWORD_QA=(qa mail password)
 DB_PASSWORD_QA=qa_password_123
 ```
 #### Production Secrets
 ```
 PROXMOX_HOST=10.0.10.95
 PROXMOX_USER=poteapp
 PROXMOX_SSH_KEY=(prod SSH key)
 SMTP_PASSWORD=(prod mail password)
 DB_PASSWORD=changeme123
 ```
 ---
 ## 📊 Deployment Flow Diagram
 ```
 Developer
    │
    ├─> Commit to feature branch
    │
    ├─> Create PR to dev
    │       │
    │       ├─> CI runs (tests)
    │       │
    │       ├─> Merge to dev
    │       │
    │       └─> Auto-deploy to DEV environment
    │
    ├─> Test in DEV
    │
    ├─> Create PR: dev → qa
    │       │
    │       ├─> CI runs (tests)
    │       │
    │       ├─> Code review required
    │       │
    │       ├─> Merge to qa
    │       │
    │       └─> Auto-deploy to QA environment
    │
    ├─> QA Testing
    │
    └─> Create PR: qa → main
            │
            ├─> CI runs (tests)
            │
            ├─> Code review required (2 approvals)
            │
            ├─> Manual approval for prod deploy
            │
            ├─> Merge to main
            │
            ├─> Create release tag
            │
            └─> Deploy to PRODUCTION
 ```
 ---
 ## 🔄 Integration with Your Ansible System
 ### Option 1: Gitea Webhooks → Ansible
 **Setup:**
 1. **In Gitea:** Settings → Webhooks → Add Webhook
   - URL: `https://your-ansible-controller/webhook/pote`
   - Trigger: Push events
   - Branches: `dev`, `qa`, `main`
 2. **Ansible Playbook:** `deploy-pote.yml`
 ```yaml
 ---
 - name: Deploy POTE based on branch
  hosts: "{{ target_env }}"
  vars:
    branch: "{{ git_branch }}"
    env: "{{ target_env }}"
  tasks:
    - name: Pull latest code
      git:
        repo: gitea@10.0.30.169:ilia/POTE.git
        dest: /home/poteapp/pote
        version: "{{ branch }}"
        force: yes
    - name: Install dependencies
      pip:
        requirements: /home/poteapp/pote/requirements.txt
        virtualenv: /home/poteapp/pote/venv
    - name: Run migrations
      command: alembic upgrade head
      args:
        chdir: /home/poteapp/pote
      environment:
        DATABASE_URL: "{{ database_url }}"
    - name: Update secrets
      template:
        src: env.j2
        dest: /home/poteapp/pote/.env
        mode: 0600
    - name: Health check
      command: python scripts/health_check.py
      args:
        chdir: /home/poteapp/pote
 ```
 3. **Ansible Inventory:** `inventory.yml`
 ```yaml
 all:
  children:
    development:
      hosts:
        dev-pote:
          ansible_host: 10.0.10.100
          target_env: development
          git_branch: dev
          database_url: postgresql://poteuser:dev_pass@localhost/potedb_dev
    staging:
      hosts:
        qa-pote:
          ansible_host: 10.0.10.101
          target_env: staging
          git_branch: qa
          database_url: postgresql://poteuser:qa_pass@localhost/potedb_qa
    production:
      hosts:
        prod-pote:
          ansible_host: 10.0.10.95
          target_env: production
          git_branch: main
          database_url: postgresql://poteuser:prod_pass@localhost/potedb
 ```
 ### Option 2: Gitea Actions → Ansible
 **File:** `.github/workflows/ansible-deploy.yml`
 ```yaml
 name: Ansible Deploy
 on:
  push:
    branches: [main, qa, dev]
 jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - name: Determine environment
        id: env
        run: |
          if [ "${{ github.ref }}" == "refs/heads/main" ]; then
            echo "environment=production" >> $GITHUB_OUTPUT
            echo "host=10.0.10.95" >> $GITHUB_OUTPUT
          elif [ "${{ github.ref }}" == "refs/heads/qa" ]; then
            echo "environment=staging" >> $GITHUB_OUTPUT
            echo "host=10.0.10.101" >> $GITHUB_OUTPUT
          else
            echo "environment=development" >> $GITHUB_OUTPUT
            echo "host=10.0.10.100" >> $GITHUB_OUTPUT
          fi
      - name: Trigger Ansible
        run: |
          curl -X POST https://your-ansible-controller/api/deploy \
            -H "Authorization: Bearer ${{ secrets.ANSIBLE_TOKEN }}" \
            -d '{
              "project": "pote",
              "environment": "${{ steps.env.outputs.environment }}",
              "branch": "${{ github.ref_name }}",
              "commit": "${{ github.sha }}"
            }'
 ```
 ---
 ## ✅ Complete Setup Checklist
 ### 1. Git Configuration
 - [ ] Create `dev` branch
 - [ ] Create `qa` branch
 - [ ] Keep `main` branch
 - [ ] Push all branches to Gitea
 ### 2. Gitea Branch Protection
 - [ ] Protect `main` (require PR + approval + CI)
 - [ ] Protect `qa` (require PR + CI)
 - [ ] Configure `dev` (require CI only)
 ### 3. Gitea Secrets
 - [ ] Add DEV environment secrets
 - [ ] Add QA environment secrets
 - [ ] Add PROD environment secrets
 ### 4. Workflows
 - [ ] Update `ci.yml` for all branches
 - [ ] Create `deploy-dev.yml`
 - [ ] Create `deploy-qa.yml`
 - [ ] Create `deploy-prod.yml`
 ### 5. Ansible Integration
 - [ ] Configure Gitea webhooks (if using webhooks)
 - [ ] Update Ansible playbooks for POTE
 - [ ] Test deployment to each environment
 ### 6. Documentation
 - [ ] Document deployment process
 - [ ] Create runbook for rollbacks
 - [ ] Train team on workflow
 ---
 ## 🚨 What You're Missing (Important!)
 ### 1. **Environment Variables per Environment**
 Create separate `.env` files:
 - `.env.dev`
 - `.env.qa`
 - `.env.prod`
 **Never commit these!** Use Ansible templates or Gitea secrets.
 ### 2. **Database Migrations Strategy**
 ```bash
 # Test migrations in dev first
 alembic upgrade head  # dev
 # Then qa
 alembic upgrade head  # qa
 # Finally prod (with backup!)
 pg_dump potedb > backup.sql
 alembic upgrade head  # prod
 ```
 ### 3. **Rollback Strategy**
 ```bash
 # Git rollback
 git checkout <previous-commit>
 # Database rollback
 alembic downgrade -1
 # Or restore from backup
 psql potedb < backup.sql
 ```
 ### 4. **Monitoring & Alerts**
 - Health checks after each deployment
 - Email/Slack notifications on failure
 - Automated rollback on critical errors
 ### 5. **Feature Flags**
 Consider adding feature flags for gradual rollouts:
 ```python
 # In config.py
 FEATURE_NEW_ANALYTICS = os.getenv("FEATURE_NEW_ANALYTICS", "false") == "true"
 ```
 ### 6. **Changelog & Release Notes**
 Maintain `CHANGELOG.md`:
 ```markdown
 ## [1.2.0] - 2025-12-15
 ### Added
 - Email reporting system
 - Gitea secrets integration
 ### Fixed
 - Database connection timeout
 ### Changed
 - Improved error handling
 ```
 ---
 ## 📚 Quick Reference Commands
 ```bash
 # Create branches
 git checkout -b dev && git push origin dev
 git checkout -b qa && git push origin qa
 # Merge dev → qa
 git checkout qa
 git merge dev
 git push origin qa
 # Merge qa → main (via PR in Gitea!)
 # Don't do this directly - use Pull Request
 # Check which branch you're on
 git branch
 # See all branches
 git branch -a
 # Deploy manually (if Ansible fails)
 ssh poteapp@10.0.10.100 "cd ~/pote && git pull origin dev"
 ```
 ---
 ## 🎯 Recommended Next Steps
 1. **Right now:** Create dev and qa branches
 2. **Today:** Set up branch protection in Gitea
 3. **This week:** Create environment-specific workflows
 4. **This week:** Integrate with your Ansible system
 5. **Next week:** Test full deployment flow
 ---
 **With this setup, you'll have a professional, production-ready deployment pipeline!** 🚀
@@ -1,353 +0,0 @@
 # ✅ Branch Strategy Setup Complete!
 ## 🌳 Branches Created
 Your POTE repository now has three branches:
 ```
 ✅ main  (production) - PROTECTED
 ✅ qa    (staging)    - Ready to protect
 ✅ dev   (development) - Ready to protect
 ```
 **Current status:**
 - `main` is already protected (you saw the error - that's good!)
 - New documentation committed to `dev` branch
 - Ready to configure protection for `qa` and `dev`
 ---
 ## 🔒 Next Steps: Configure Branch Protection
 ### Go to Gitea: https://git.levkin.ca/ilia/POTE/settings/branches
 ### 1. Protect `main` (Production) - Already Done! ✅
 Your `main` branch is already protected (we couldn't push directly to it).
 **Verify settings:**
 - Branch name pattern: `main`
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require approvals: 1 (or 2 for production)
 - ✅ Require status checks
 - ✅ Block force push
 - ✅ Block deletion
 ### 2. Protect `qa` (Staging) - TODO
 Click "Add New Rule":
 - Branch name pattern: `qa`
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require 1 approval
 - ✅ Require status checks to pass
 - ✅ Block force push
 - ✅ Block deletion
 ### 3. Configure `dev` (Development) - TODO
 Click "Add New Rule":
 - Branch name pattern: `dev`
 - ✅ Require status checks to pass (CI must pass)
 - ⚠️ Allow direct push (for rapid development)
 - ✅ Block force push (optional)
 ---
 ## 📋 What You're Missing (Checklist)
 ### ✅ Already Have:
 - [x] Three branches (main, qa, dev)
 - [x] Main branch protection
 - [x] Comprehensive documentation
 - [x] CI/CD pipeline
 - [x] Gitea secrets integration
 ### 🔲 Need to Add:
 #### 1. **Environment-Specific Secrets in Gitea**
 Go to: https://git.levkin.ca/ilia/POTE/settings/secrets
 **Development:**
 ```
 DEV_HOST=10.0.10.100 (or your dev server IP)
 DEV_USER=poteapp
 DEV_SSH_KEY=(SSH key for dev server)
 SMTP_PASSWORD_DEV=(dev email password)
 DB_PASSWORD_DEV=dev_password_123
 ```
 **QA/Staging:**
 ```
 QA_HOST=10.0.10.101 (or your QA server IP)
 QA_USER=poteapp
 QA_SSH_KEY=(SSH key for QA server)
 SMTP_PASSWORD_QA=(qa email password)
 DB_PASSWORD_QA=qa_password_123
 ```
 **Production:**
 ```
 PROXMOX_HOST=10.0.10.95 (already have this)
 PROXMOX_USER=poteapp
 PROXMOX_SSH_KEY=(already have this)
 SMTP_PASSWORD=(already have this)
 DB_PASSWORD=changeme123
 ```
 #### 2. **Create Environment-Specific Deployment Workflows**
 Files to create:
 - `.github/workflows/deploy-dev.yml` (see docs/14_branch_strategy_and_deployment.md)
 - `.github/workflows/deploy-qa.yml`
 - `.github/workflows/deploy-prod.yml` (already have deploy.yml, can rename/update)
 #### 3. **Set Up Separate Servers/Containers**
 You need three environments:
 | Environment | Server/Container | Database | Purpose |
 |-------------|------------------|----------|---------|
 | **Dev** | `10.0.10.100` (or new LXC) | `potedb_dev` | Development testing |
 | **QA** | `10.0.10.101` (or new LXC) | `potedb_qa` | Pre-production testing |
 | **Prod** | `10.0.10.95` (existing) | `potedb` | Production |
 **Options:**
 - Create 2 more LXC containers (recommended)
 - Use same server with different ports/databases
 - Use Docker containers
 #### 4. **Ansible Integration**
 **Option A: Gitea Webhooks**
 ```
 Gitea → Settings → Webhooks → Add Webhook
 URL: https://your-ansible-controller/webhook/pote
 Trigger on: Push events
 Branches: dev, qa, main
 ```
 **Option B: Gitea Actions calls Ansible**
 ```yaml
 # In workflow
 - name: Trigger Ansible
  run: |
    curl -X POST https://ansible-controller/api/deploy \
      -d '{"branch": "${{ github.ref_name }}"}'
 ```
 #### 5. **Update Ansible Playbook**
 Your Ansible playbook should:
 ```yaml
 - name: Deploy POTE
  hosts: "{{ target_env }}"
  vars:
    branch: "{{ git_branch }}"  # dev, qa, or main
  tasks:
    - git:
        repo: gitea@10.0.30.169:ilia/POTE.git
        dest: /home/poteapp/pote
        version: "{{ branch }}"
    # ... rest of deployment
 ```
 #### 6. **Database Migration Strategy**
 ```bash
 # Always test in dev first
 ssh poteapp@dev-server "cd ~/pote && alembic upgrade head"
 # Then QA
 ssh poteapp@qa-server "cd ~/pote && alembic upgrade head"
 # Finally prod (with backup!)
 ssh poteapp@prod-server "pg_dump potedb > backup.sql && cd ~/pote && alembic upgrade head"
 ```
 #### 7. **Monitoring & Alerts**
 Add to each deployment:
 ```yaml
 - name: Health Check
  run: python scripts/health_check.py
 - name: Send Alert on Failure
  if: failure()
  run: |
    # Send email/Slack notification
 ```
 #### 8. **Environment Variables**
 Create separate configs:
 - `.env.dev` (in dev server)
 - `.env.qa` (in qa server)
 - `.env` (in prod server - already have)
 **Never commit these!** Use Ansible templates or deployment workflows.
 ---
 ## 🚀 Workflow After Setup
 ### Development Flow:
 ```bash
 # 1. Work on feature
 git checkout dev
 git pull origin dev
 # ... make changes ...
 git commit -m "Add feature"
 git push origin dev
 # 2. Auto-deploys to DEV server
 # (via Gitea webhook or Actions)
 # 3. Test in DEV
 # 4. Promote to QA
 # Create PR: dev → qa in Gitea UI
 # Merge after approval
 # Auto-deploys to QA server
 # 5. QA Testing
 # 6. Promote to PROD
 # Create PR: qa → main in Gitea UI
 # Requires 2 approvals
 # Merge
 # Manual deployment trigger (with confirmation)
 ```
 ---
 ## 📚 Documentation
 **Main guide:** `docs/14_branch_strategy_and_deployment.md`
 Covers:
 - ✅ Branch protection setup
 - ✅ Multi-environment workflows
 - ✅ Ansible integration
 - ✅ Deployment flow
 - ✅ Rollback procedures
 - ✅ Complete checklist
 ---
 ## 🎯 Quick Actions (Do These Now)
 ### 1. Configure Branch Protection (5 minutes)
 ```
 https://git.levkin.ca/ilia/POTE/settings/branches
 - Add rule for 'qa'
 - Add rule for 'dev'
 ```
 ### 2. Add Environment Secrets (10 minutes)
 ```
 https://git.levkin.ca/ilia/POTE/settings/secrets
 - Add DEV_* secrets
 - Add QA_* secrets
 - Verify PROD secrets exist
 ```
 ### 3. Create PR for Documentation (2 minutes)
 ```
 https://git.levkin.ca/ilia/POTE/compare/main...dev
 - Create pull request
 - Title: "Add branch strategy documentation"
 - Merge to main
 ```
 ### 4. Decide on Server Setup
 **Option 1:** Create 2 more LXC containers
 ```bash
 # On Proxmox host
 pct clone 100 101 --hostname pote-dev
 pct clone 100 102 --hostname pote-qa
 ```
 **Option 2:** Use existing server with different databases
 ```bash
 # On existing server
 createdb potedb_dev
 createdb potedb_qa
 ```
 ### 5. Configure Ansible
 Update your Ansible inventory to include:
 - `pote-dev` host
 - `pote-qa` host
 - `pote-prod` host (existing)
 ---
 ## ⚠️ Important Notes
 ### Main Branch is Protected!
 You saw this error:
 ```
 remote: Gitea: Not allowed to push to protected branch main
 ```
 **This is GOOD!** It means:
 - ✅ Main branch is protected
 - ✅ Can't accidentally push directly
 - ✅ Must use Pull Requests
 - ✅ Requires code review
 **To update main:**
 1. Push to `dev` or `qa`
 2. Create Pull Request in Gitea
 3. Get approval
 4. Merge
 ### Current Branch Status
 ```bash
 $ git branch
  dev    ← New documentation is here
 * main   ← Protected, can't push directly
  qa     ← Empty, same as main
 ```
 ---
 ## 🔗 Links
 - **Repository:** https://git.levkin.ca/ilia/POTE
 - **Branch Protection:** https://git.levkin.ca/ilia/POTE/settings/branches
 - **Secrets:** https://git.levkin.ca/ilia/POTE/settings/secrets
 - **Actions:** https://git.levkin.ca/ilia/POTE/actions
 - **Create PR:** https://git.levkin.ca/ilia/POTE/compare/main...dev
 ---
 ## ✅ Summary
 **What's Done:**
 - ✅ Created `dev`, `qa`, `main` branches
 - ✅ Main branch is protected
 - ✅ Documentation committed to `dev`
 - ✅ Ready for Ansible integration
 **What's Next:**
 1. Configure branch protection for `qa` and `dev`
 2. Add environment-specific secrets
 3. Create PR to merge docs to main
 4. Set up dev/qa servers
 5. Configure Ansible for multi-environment
 6. Test deployment flow
 **You're 80% there! Just need to configure Gitea settings and set up the additional servers.** 🚀
@@ -1,378 +0,0 @@
 # 🔧 Pipeline Setup Guide for Branch Protection
 ## ❓ Do You Need a Pipeline?
 **YES!** If you want to use "Require status checks" in branch protection, you need a CI pipeline.
 **Good news:** You already have one! ✅
 ---
 ## ✅ What You Already Have
 ### CI Pipeline: `.github/workflows/ci.yml`
 **Status:** ✅ Exists and working
 **Runs on:** Push to `main`, `qa`, `dev` (just updated!)
 **What it does:**
 - Runs linters (ruff, black, mypy)
 - Runs 93 tests
 - Checks code quality
 - Uses PostgreSQL for integration tests
 ---
 ## 🚀 Setup Order (IMPORTANT!)
 ### ⚠️ **DO THIS IN ORDER:**
 ### Step 1: Merge CI Updates to Main (FIRST!)
 **Why:** Branch protection needs the CI pipeline to exist in the branch you're protecting.
 **How:**
 1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
 2. Click "New Pull Request"
 3. Title: "Update CI for multi-branch support"
 4. **Merge this PR** (you can merge without protection for now)
 **What this does:**
 - Updates CI to run on `main`, `qa`, and `dev`
 - Makes CI available for branch protection
 ---
 ### Step 2: Verify CI is Working
 After merging the PR, check:
 1. Go to: https://git.levkin.ca/ilia/POTE/actions
 2. You should see the CI workflow running
 3. Wait for it to complete (green checkmark ✅)
 **If CI fails:**
 - Don't set up branch protection yet
 - Fix the CI issues first
 - Ensure tests pass
 ---
 ### Step 3: Configure Branch Protection (AFTER CI WORKS)
 **Only after CI is passing**, go to:
 https://git.levkin.ca/ilia/POTE/settings/branches
 #### For `main` Branch (Already Protected)
 **Verify these settings:**
 - Branch pattern: `main`
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require 1-2 approvals
 - ✅ **Require status checks to pass before merging**
  - Select: `CI / lint-and-test` (this appears after CI runs once)
 - ✅ Block force push
 - ✅ Block deletion
 #### For `qa` Branch
 Click "Add New Rule":
 - Branch pattern: `qa`
 - ✅ Enable push protection
 - ✅ Require pull request
 - ✅ Require 1 approval
 - ✅ **Require status checks to pass before merging**
  - Select: `CI / lint-and-test`
 - ✅ Block force push
 - ✅ Block deletion
 #### For `dev` Branch
 Click "Add New Rule":
 - Branch pattern: `dev`
 - ✅ **Require status checks to pass before merging**
  - Select: `CI / lint-and-test`
 - ⚠️ **Allow direct push** (no PR required for dev)
 - ⚠️ Allow force push (optional, for rebasing)
 ---
 ## 🔍 What "Require Status Checks" Means
 When you enable "Require status checks":
 **Before merge:**
 ```
 PR created: dev → qa
    ↓
 CI pipeline runs automatically
    ↓
 Tests must pass ✅
    ↓
 Only then can you merge
 ```
 **If CI fails:**
 ```
 PR created: dev → qa
    ↓
 CI pipeline runs
    ↓
 Tests fail ❌
    ↓
 Merge button is DISABLED
    ↓
 Must fix code and push again
 ```
 ---
 ## 📋 Status Checks Available
 After your CI runs once, you'll see these options in branch protection:
 **Available checks:**
 - `CI / lint-and-test` - Main CI pipeline (93 tests)
 - `CI / security-scan` - Security scanning
 - `CI / dependency-scan` - Dependency vulnerabilities
 - `CI / docker-build-test` - Docker build verification
 **Recommended:**
 - **For `main`:** Require ALL checks ✅
 - **For `qa`:** Require `lint-and-test` + `security-scan` ✅
 - **For `dev`:** Require `lint-and-test` only ✅
 ---
 ## 🎯 Step-by-Step Setup (Complete)
 ### 1. Merge CI Updates (5 minutes)
 ```
 1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
 2. Create PR: "Update CI for multi-branch support"
 3. Merge (you can approve your own PR for now)
 4. Wait for CI to run on main branch
 ```
 ### 2. Check CI Status (2 minutes)
 ```
 1. Go to: https://git.levkin.ca/ilia/POTE/actions
 2. Click on the latest workflow run
 3. Verify all jobs pass ✅
 ```
 ### 3. Configure Branch Protection (10 minutes)
 ```
 1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
 2. For main (update existing rule):
   - ✅ Require status checks
   - Select: CI / lint-and-test
 3. Add rule for qa:
   - Branch: qa
   - ✅ Require PR
   - ✅ Require status checks
   - Select: CI / lint-and-test
 4. Add rule for dev:
   - Branch: dev
   - ✅ Require status checks
   - Select: CI / lint-and-test
   - ⚠️ Allow direct push
 ```
 ---
 ## ⚠️ Common Issues
 ### Issue 1: "No status checks found"
 **Cause:** CI hasn't run on that branch yet
 **Fix:**
 ```bash
 # Push something to trigger CI
 git checkout dev
 git commit --allow-empty -m "Trigger CI"
 git push origin dev
 # Wait for CI to run, then configure protection
 ```
 ### Issue 2: "Status check never completes"
 **Cause:** CI is failing or stuck
 **Fix:**
 1. Go to Actions tab
 2. Check the failing job
 3. Fix the issue
 4. Push again
 ### Issue 3: "Can't select status checks in dropdown"
 **Cause:** CI workflow name doesn't match
 **Fix:**
 - Workflow must be named exactly: `CI`
 - Job must be named: `lint-and-test`
 - Already correct in your `.github/workflows/ci.yml` ✅
 ---
 ## 🧪 Test Your Setup
 ### After configuring protection:
 **Test 1: Try to push directly to main**
 ```bash
 git checkout main
 git commit --allow-empty -m "Test"
 git push origin main
 # Should fail: "Not allowed to push to protected branch"
 ```
 **Test 2: Create PR with failing tests**
 ```bash
 git checkout dev
 # Break a test intentionally
 git commit -m "Break test"
 git push origin dev
 # Create PR to qa
 # Merge button should be disabled until CI passes
 ```
 **Test 3: Create PR with passing tests**
 ```bash
 git checkout dev
 # Fix the test
 git commit -m "Fix test"
 git push origin dev
 # Create PR to qa
 # Merge button should be enabled after CI passes ✅
 ```
 ---
 ## 📊 What Happens After Setup
 ### Workflow with Protection:
 ```
 Developer pushes to dev
    ↓
 CI runs automatically
    ↓
 ✅ Tests pass
    ↓
 Developer creates PR: dev → qa
    ↓
 CI runs on PR
    ↓
 ✅ Tests pass
    ↓
 Reviewer approves
    ↓
 ✅ Merge button enabled
    ↓
 Merge to qa
    ↓
 CI runs on qa branch
    ↓
 ✅ Tests pass
    ↓
 Auto-deploy to QA server (if configured)
 ```
 **If tests fail at any point:**
 ```
 CI runs
    ↓
 ❌ Tests fail
    ↓
 Merge button DISABLED
    ↓
 Developer fixes code
    ↓
 Pushes again
    ↓
 CI runs again
    ↓
 Loop until tests pass ✅
 ```
 ---
 ## ✅ Checklist
 Before configuring branch protection:
 - [ ] CI workflow exists (`.github/workflows/ci.yml`) ✅
 - [ ] CI runs on all branches (`main`, `qa`, `dev`) ✅
 - [ ] CI has run at least once on each branch
 - [ ] All tests are passing ✅
 - [ ] You can see workflow runs in Actions tab
 After configuring:
 - [ ] `main` branch requires status checks
 - [ ] `qa` branch requires status checks
 - [ ] `dev` branch requires status checks
 - [ ] Tested: Can't push directly to `main`
 - [ ] Tested: PR merge blocked when CI fails
 - [ ] Tested: PR merge allowed when CI passes
 ---
 ## 🎯 Quick Start (TL;DR)
 ```bash
 # 1. Merge CI updates
 # Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
 # Create and merge PR
 # 2. Wait for CI to run
 # Check: https://git.levkin.ca/ilia/POTE/actions
 # 3. Configure branch protection
 # Go to: https://git.levkin.ca/ilia/POTE/settings/branches
 # Add rules for main, qa, dev
 # Enable "Require status checks"
 # Select "CI / lint-and-test"
 # Done! ✅
 ```
 ---
 ## 📚 Related Documentation
 - **CI Workflow:** `.github/workflows/ci.yml`
 - **Branch Strategy:** `docs/14_branch_strategy_and_deployment.md`
 - **Setup Checklist:** `BRANCH_SETUP_COMPLETE.md`
 - **Gitea Secrets:** `GITEA_SECRETS_GUIDE.md`
 ---
 ## 🚀 Summary
 **Do you need a pipeline?**
 - ✅ YES, to use "Require status checks"
 - ✅ You already have one!
 - ✅ Just need to merge it to main first
 **Setup order:**
 1. Merge CI updates to main (via PR)
 2. Verify CI runs and passes
 3. Configure branch protection
 4. Test the protection
 **After setup:**
 - All branches protected by CI
 - Can't merge failing code
 - Professional development workflow
 - Ready for Ansible integration
 **You're almost there! Just merge the PR and configure protection.** 🎉
@@ -0,0 +1,163 @@
 # POTE homelab handoff — 2026-05-27
 **Status:** Production LXC running; PR #1 merged to `main`; CI green on Gitea Actions.  
 **Research only — not investment advice.**
 ---
 ## What’s live
 | Item | Value |
 |------|--------|
 | Host | LXC **236** `pote` @ **10.0.10.48** (pve10) |
 | App | `/home/poteapp/pote` (venv, **no git clone** — deploy via rsync) |
 | DB | PostgreSQL `pote` / `poteuser` (password rotated; in Ansible vault) |
 | Data | ~55 officials, ~329 trades (30-day live ingest, May 2026) |
 | SMTP | `10.0.10.132` (Mailcow), send as **`alerts@levkine.ca`** |
 | Reports | **`idobkin@gmail.com`** daily 07:00, weekly Sun 08:00 |
 ### Cron (`crontab -u poteapp -l`)
 | Time | Script |
 |------|--------|
 | 06:00 | `fetch_congressional_trades.py --days 7` |
 | 06:15 | `enrich_securities.py` |
 | 06:30 | `monitor_market.py --scan` |
 | 07:00 | `send_daily_report.py --to idobkin@gmail.com` |
 | Sun 08:00 | `send_weekly_report.py --to idobkin@gmail.com` |
 ### Data source (important)
 Legacy **housestockwatcher.com** and S3 buckets are dead/blocked. Ingest uses public JSON from [congress-trading-monitor](https://github.com/kadoa-org/congress-trading-monitor) (~5000 rows cap). Override with env `POTE_HOUSE_DATA_URL` if you add another feed.
 ---
 ## Repos & branches
 | Repo | Branch | Notes |
 |------|--------|--------|
 | **POTE** | `main` @ `git.levkin.ca/ilia/POTE` | Merged PR #1 — ingest, email, CI, deps |
 | **ansible** | `feature/outline-setup-api` (or `master`) | Inventory, `deploy-pote.sh`, vault — may need merge to homelab default branch |
 Local:
 ```bash
 cd ~/Documents/code/POTE && git checkout main && git pull
 ```
 ---
 ## Quick access
 ```bash
 ssh root@10.0.10.48
 su - poteapp
 cd pote && source venv/bin/activate
 # Logs
 tail -f ~/logs/daily_report.log
 tail -f ~/logs/trades.log
 # Manual run
 python scripts/fetch_congressional_trades.py --days 30
 python scripts/send_daily_report.py --to idobkin@gmail.com --test-smtp
 ```
 Deploy code from laptop (preserves server `.env`):
 ```bash
 cd ~/Documents/code/ansible
 make deploy-pote
 # or: RUN_FETCH=1 make deploy-pote
 ```
 ---
 ## Ansible / homelab inventory
 Already wired (ansible repo):
 - `inventories/production/hosts` — `pote` @ `.48`, VMID 236
 - `docs/guides/host-list.md` — LXC 236 row
 - `scripts/beszel-install-agents.sh` — `pote-236`
 - `scripts/deploy-pote.sh`, `make deploy-pote`
 - `scripts/vault-update-pote.py`, `make vault-update-pote`
 - `docs/guides/smtp-inventory.md` — POTE uses `alerts@levkine.ca`
 - Vault: `vault_pote_db_password_prod`, `vault_pote_smtp_password`
 ```bash
 make vault-export-env
 make beszel-install-agents BESZEL_ONLY=pote-236   # if agent not yet installed
 ```
 ---
 ## Verify after first automated day
 1. **07:00+** — Email in Gmail (From: `alerts@levkine.ca`, subject `POTE Daily Report - YYYY-MM-DD`). Check spam once.
 2. **Logs** — `~/logs/daily_report.log`, `trades.log` — no tracebacks.
 3. **DB growth** — trade count should tick up on weekdays:
   ```bash
   su - poteapp -c 'cd pote && source venv/bin/activate && python -c "
   from sqlalchemy import func, select
   from pote.db import SessionLocal
   from pote.db.models import Trade, Official
   with SessionLocal() as s:
       print(\"trades\", s.scalar(select(func.count(Trade.id))))
       print(\"officials\", s.scalar(select(func.count(Official.id))))
   "'
   ```
 ---
 **Vikunja:** [todo.levkin.ca → Business → POTE](https://todo.levkin.ca) (`POTE`)
 ## Open tasks (source of truth)
 | P | Task | Owner | Status |
 |---|------|-------|--------|
 | **P1** | Proxmox backup schedule for LXC **236** on pve10 | @you | ⏳ |
 | **P3** | Dedicated `pote@levkine.ca` mailbox (vs shared `alerts@`) | @you | optional / low |
 | **P3** | Git deploy on LXC (replace rsync-only) | @agent | optional / low |
 | **P3** | Kuma LAN health monitor | @agent | optional / low |
 | **P3** | Mattermost / webhook alerts (email only today) | @agent | optional / low |
 | **P3** | Full history ingest (second data source beyond kadoa cap) | @agent | optional / low |
 **Closed 2026-07-11:** Beszel agent on pote-236 (`beszel-agent` active).  
 **Closed 2026-07-12:** `make deploy-pote` synced to LXC 236; 07:00 cron email delivered (`idobkin@gmail.com`, log OK). DB ~1129 trades / 80 officials.
 ### Not planned (unless you want them)
 - Public URL / Caddy vhost (LAN-only by design)
 - Investment signals exposed as advice (research descriptors only)
 ---
 ## Known issues / caveats
 | Topic | Detail |
 |-------|--------|
 | **Disclosure lag** | STOCK Act filings appear weeks after trades; reports are descriptive, not timely trading signals. |
 | **Amount ranges** | Disclosure buckets only ($1k–$15k, etc.), not exact sizes. |
 | **Empty tickers** | Some filings skipped when ticker missing. |
 | **CI vs prod** | CI uses venv + Postgres service; prod uses host Postgres — both should pass after merge. |
 | **Gitea deploy workflow** | `.github/workflows/deploy.yml` still references `git pull` on Proxmox; prod uses **rsync** via ansible `deploy-pote.sh`. |
 ---
 ## Related docs
 | Doc | Purpose |
 |-----|---------|
 | [EMAIL_SETUP.md](../EMAIL_SETUP.md) | SMTP / Mailcow / levkine.ca |
 | [AUTOMATION_QUICKSTART.md](../AUTOMATION_QUICKSTART.md) | Cron + reports |
 | [PROXMOX_QUICKSTART.md](../PROXMOX_QUICKSTART.md) | Original LXC provisioning |
 | Ansible `docs/guides/projects-handoff-2026-05-26.md` | Multi-project homelab context |
 | Ansible `docs/guides/smtp-inventory.md` | Mailboxes |
 ---
 ## One-line summary
 **POTE on 10.0.10.48 ingests public congressional trades daily, emails a research summary to Gmail at 07:00, and is maintained via `main` + `make deploy-pote` — verify tomorrow’s cron email, then Beszel, backups, and optional data-source expansion.**
@@ -18,8 +18,10 @@ dependencies = [
    "pydantic-settings>=2.0",
    "python-dotenv>=1.0",
    "requests>=2.31",
    "httpx>=0.27",
    "pandas>=2.0",
    "numpy>=1.24",
    "scikit-learn>=1.3",
    "yfinance>=0.2",
    "psycopg2-binary>=2.9",
 ]
@@ -28,8 +28,8 @@ def main():
    args = parser.parse_args()
-    logger.info("=== Fetching Congressional Trades from House Stock Watcher ===")
+    logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===")
-    logger.info("Source: https://housestockwatcher.com (free, no API key)")
+    logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)")
    try:
        with HouseWatcherClient() as client:
@@ -0,0 +1,46 @@
 #!/usr/bin/env bash
 # One-time installer for the local pre-commit gitleaks hook.
 # Run once per clone: bash scripts/git-hooks/install.sh
 #
 # Respects `core.hooksPath` if you've set one (local or global) — some setups
 # point git at a hooks dir outside `.git/hooks/` (e.g. a machine-wide
 # `~/.git-hooks/`), and installing to `.git/hooks/` in that case would be a
 # silent no-op. If an existing hook is already at that path, this chains to
 # it so nothing already relying on it breaks.
 set -euo pipefail
 REPO_ROOT="$(git rev-parse --show-toplevel)"
 cd "$REPO_ROOT"
 HOOKS_DIR="$(git config --get core.hooksPath || true)"
 if [ -z "$HOOKS_DIR" ]; then
  HOOKS_DIR=".git/hooks"
 elif [[ "$HOOKS_DIR" != /* ]]; then
  HOOKS_DIR="$REPO_ROOT/$HOOKS_DIR"
 fi
 mkdir -p "$HOOKS_DIR"
 TARGET="$HOOKS_DIR/pre-commit"
 if [ -f "$TARGET" ] && ! grep -q "gitleaks" "$TARGET" 2>/dev/null; then
  echo "⚠️  Existing pre-commit hook found at $TARGET that isn't ours — chaining instead of overwriting."
  CHAINED="$HOOKS_DIR/pre-commit.d-gitleaks"
  cp scripts/git-hooks/pre-commit "$CHAINED"
  chmod +x "$CHAINED"
  if ! grep -q "pre-commit.d-gitleaks" "$TARGET" 2>/dev/null; then
    printf '\n# Added by levkinops ansible repo (scripts/git-hooks/install.sh)\n"%s"\n' "$CHAINED" >> "$TARGET"
  fi
 else
  cp scripts/git-hooks/pre-commit "$TARGET"
  chmod +x "$TARGET"
 fi
 echo "✓ Installed pre-commit gitleaks hook → $TARGET"
 if [ "$HOOKS_DIR" != ".git/hooks" ] && [ "$HOOKS_DIR" != "$REPO_ROOT/.git/hooks" ]; then
  echo "  (using core.hooksPath=$HOOKS_DIR — applies to every repo that shares this hooksPath)"
 fi
 if ! command -v gitleaks >/dev/null 2>&1; then
  echo "  Note: gitleaks isn't installed locally yet. Install it for the hook to actually run:"
  echo "    macOS:  brew install gitleaks"
  echo "    Linux:  see https://github.com/gitleaks/gitleaks#installing"
  echo "  Until then this hook is a no-op locally (CI still scans every push)."
 fi
@@ -0,0 +1,27 @@
 #!/usr/bin/env bash
 # Local pre-commit secret scan — mirrors the `secret-scanning` CI job (gitleaks)
 # so leaked secrets are caught before they ever leave your machine, not just at
 # CI time. Installed via `make install-git-hooks`.
 #
 # Uses the same .gitleaks.toml allowlist as CI. Only scans staged content.
 set -euo pipefail
 REPO_ROOT="$(git rev-parse --show-toplevel)"
 cd "$REPO_ROOT"
 if ! command -v gitleaks >/dev/null 2>&1; then
  echo "⚠️  gitleaks not installed locally — skipping local secret scan."
  echo "    Install: brew install gitleaks   (CI will still catch it either way)"
  exit 0
 fi
 echo "🔐 Running gitleaks on staged changes..."
 if ! gitleaks protect --staged --config .gitleaks.toml --no-banner --redact; then
  echo ""
  echo "❌ gitleaks found a potential secret in your staged changes."
  echo "   Fix it, or if it's a false positive, add an allowlist entry to .gitleaks.toml."
  echo "   Bypass (not recommended): git commit --no-verify"
  exit 1
 fi
 echo "✓ gitleaks: no secrets found in staged changes"
@@ -43,6 +43,12 @@ def main():
        action="store_true",
        help="Test SMTP connection before sending",
    )
    parser.add_argument(
        "--lookback-days",
        type=int,
        default=1,
        help="Include trades filed in the last N days ending on the report date (default: 1)",
    )
    parser.add_argument(
        "--save-to-file",
        help="Also save report to this file path",
@@ -81,7 +87,9 @@ def main():
    logger.info(f"Generating daily report for {report_date or date.today()}...")
    with get_session() as session:
        generator = ReportGenerator(session)
-        report_data = generator.generate_daily_summary(report_date)
+        report_data = generator.generate_daily_summary(
            report_date, lookback_days=args.lookback_days
        )
        # Format as text and HTML
        text_body = generator.format_as_text(report_data, "daily")
@@ -30,6 +30,16 @@ class Settings(BaseSettings):
    # Logging
    log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)")
    # Email (Mailcow @ mail.levkine.ca — use LAN IP from app LXCs if DNS points public)
    smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname")
    smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)")
    smtp_user: str = Field(default="", description="SMTP auth username")
    smtp_password: str = Field(default="", description="SMTP auth password")
    from_email: str = Field(default="", description="From address for reports")
    report_recipients: str = Field(
        default="", description="Default report recipients (comma-separated)"
    )
    # Application
    app_name: str = "POTE"
    app_version: str = "0.1.0"
@@ -3,6 +3,7 @@ Database layer: engine, session factory, and base model.
 """
 from collections.abc import Generator
 from contextlib import contextmanager
 from sqlalchemy import create_engine
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
@@ -26,8 +27,9 @@ class Base(DeclarativeBase):
    pass
@contextmanager
 def get_session() -> Generator[Session, None, None]:
-    """Get a database session (use as a context manager or dependency)."""
+    """Get a database session (context manager or FastAPI-style dependency)."""
    session = SessionLocal()
    try:
        yield session
@@ -1,9 +1,12 @@
 """
 House Stock Watcher client for fetching congressional trade data.
-Free, no API key required - scrapes from housestockwatcher.com
+
 Uses public STOCK Act disclosure datasets (no API key). The legacy
 housestockwatcher.com host is often unavailable; we try several mirrors.
 """
 import logging
 import os
 from datetime import date, datetime
 from typing import Any
@@ -11,16 +14,69 @@ import httpx
 logger = logging.getLogger(__name__)
 def _default_data_urls() -> tuple[str, ...]:
    override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip()
    if override:
        return (override,)
    return (
        "https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json",
        "https://housestockwatcher.com/api/all_transactions",
        "https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json",
    )
 DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls()
 def _ascii_safe(text: str | None) -> str:
    """Normalize text for DB fields that may use ASCII-only client encoding."""
    if not text:
        return ""
    return text.replace("\u00b7", "-").replace("·", "-").strip()
 def _party_label(code: str | None) -> str:
    if not code:
        return ""
    mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"}
    return mapping.get(code.strip().upper(), code.strip())
 def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]:
    """
    Map a raw disclosure record to the House Stock Watcher field names
    expected by TradeLoader.
    """
    if "representative" in raw and "disclosure_date" in raw:
        return raw
    # Kadoa congress-trading-monitor export
    if "filer_name" in raw:
        chamber = (raw.get("chamber") or "").strip().lower()
        house = "Senate" if chamber == "senate" else "House"
        return {
            "representative": raw.get("filer_name", "").strip(),
            "ticker": (raw.get("ticker") or "").strip(),
            "transaction_date": raw.get("transaction_date", ""),
            "disclosure_date": raw.get("filing_date", ""),
            "transaction": raw.get("transaction_type", ""),
            "amount": raw.get("amount_range_label", ""),
            "house": house,
            "district": _ascii_safe(raw.get("office")),
            "party": _party_label(raw.get("party")),
        }
    return raw
 class HouseWatcherClient:
    """
-    Client for House Stock Watcher API (free, community-maintained).
+    Client for congressional trade JSON feeds (free, community-maintained).
-    Data source: https://housestockwatcher.com/
+    Primary source: congress-trading-monitor public dataset on GitHub.
    No authentication required.
    """
-    BASE_URL = "https://housestockwatcher.com/api"
+    data_urls: tuple[str, ...] = DEFAULT_DATA_URLS
    def __init__(self, timeout: float = 30.0):
        """
@@ -65,30 +121,31 @@ class HouseWatcherClient:
        Raises:
            httpx.HTTPError: If request fails
        """
-        url = f"{self.BASE_URL}/all_transactions"
+        last_error: Exception | None = None
-        logger.info(f"Fetching transactions from {url}")
+        for url in self.data_urls:
            if not url:
                continue
            logger.info(f"Fetching transactions from {url}")
            try:
                response = self._client.get(url)
                response.raise_for_status()
                data = response.json()
                if not isinstance(data, list):
                    raise ValueError(f"Expected list response, got {type(data)}")
                data = [normalize_transaction_record(item) for item in data]
                logger.info(f"Fetched {len(data)} transactions from {url}")
                if limit:
                    data = data[:limit]
                return data
            except Exception as e:
                last_error = e
                logger.warning(f"Failed to fetch from {url}: {e}")
                continue
-        try:
+        logger.error("Failed to fetch congressional trades from all configured URLs")
-            response = self._client.get(url)
+        if last_error:
-            response.raise_for_status()
+            raise last_error
-            data = response.json()
+        raise RuntimeError("No data URLs configured")
            if not isinstance(data, list):
                raise ValueError(f"Expected list response, got {type(data)}")
            logger.info(f"Fetched {len(data)} transactions from House Stock Watcher")
            if limit:
                data = data[:limit]
            return data
        except httpx.HTTPError as e:
            logger.error(f"Failed to fetch from House Stock Watcher: {e}")
            raise
        except Exception as e:
            logger.error(f"Unexpected error fetching transactions: {e}")
            raise
    def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]:
        """
@@ -45,27 +45,25 @@ class TradeLoader:
        for txn in transactions:
            try:
-                # Get or create official
+                with self.session.begin_nested():
-                official, is_new_official = self._get_or_create_official(txn)
+                    official, is_new_official = self._get_or_create_official(txn)
-                if is_new_official:
+                    if is_new_official:
-                    officials_created += 1
+                        officials_created += 1
-                # Get or create security
+                    ticker = txn.get("ticker", "").strip().upper()
-                ticker = txn.get("ticker", "").strip().upper()
+                    if not ticker or ticker in ("N/A", "--", ""):
-                if not ticker or ticker in ("N/A", "--", ""):
+                        logger.debug(
-                    logger.debug(
+                            f"Skipping transaction with no ticker: {txn.get('representative')}"
-                        f"Skipping transaction with no ticker: {txn.get('representative')}"
+                        )
-                    )
+                        continue
                    continue
-                security, is_new_security = self._get_or_create_security(ticker)
+                    security, is_new_security = self._get_or_create_security(ticker)
-                if is_new_security:
+                    if is_new_security:
-                    securities_created += 1
+                        securities_created += 1
-                # Create trade (upsert)
+                    trade_created = self._upsert_trade(txn, official.id, security.id, source)
-                trade_created = self._upsert_trade(txn, official.id, security.id, source)
+                    if trade_created:
-                if trade_created:
+                        trades_created += 1
                    trades_created += 1
            except Exception as e:
                logger.error(f"Failed to ingest transaction {txn}: {e}")
@@ -27,13 +27,15 @@ class ReportGenerator:
        self.detector = PatternDetector(session)
    def generate_daily_summary(
-        self, report_date: Optional[date] = None
+        self, report_date: Optional[date] = None, *, lookback_days: int = 1
    ) -> Dict[str, Any]:
        """
        Generate a daily summary report.
        Args:
            report_date: Date to generate report for (defaults to today)
            lookback_days: Include trades filed in the last N days ending on report_date
                (defaults to 1, meaning only filings on report_date).
        Returns:
            Dictionary containing report data
@@ -41,12 +43,19 @@ class ReportGenerator:
        if report_date is None:
            report_date = date.today()
        if lookback_days < 1:
            raise ValueError("lookback_days must be >= 1")
        start_of_day = datetime.combine(report_date, datetime.min.time())
        end_of_day = datetime.combine(report_date, datetime.max.time())
-        # Count new trades filed today
+        filing_start_date = report_date - timedelta(days=lookback_days - 1)
        # Trades filed within the lookback window (inclusive)
        new_trades = (
-            self.session.query(Trade).filter(Trade.filing_date == report_date).all()
+            self.session.query(Trade)
            .filter(Trade.filing_date >= filing_start_date, Trade.filing_date <= report_date)
            .all()
        )
        # Count market alerts today
@@ -71,6 +80,8 @@ class ReportGenerator:
        return {
            "date": report_date,
            "filing_start_date": filing_start_date,
            "lookback_days": lookback_days,
            "new_trades_count": len(new_trades),
            "new_trades": [
                {
@@ -173,13 +184,20 @@ class ReportGenerator:
    def _format_daily_text(self, data: Dict[str, Any]) -> str:
        """Format daily report as plain text."""
        if data.get("lookback_days", 1) > 1:
            trades_label = (
                f"  • Trades Filed (last {data['lookback_days']} days): {data['new_trades_count']}"
            )
        else:
            trades_label = f"  • New Trades Filed: {data['new_trades_count']}"
        lines = [
            "=" * 70,
            f"POTE DAILY REPORT - {data['date']}",
            "=" * 70,
            "",
            "📊 SUMMARY",
-            f"  • New Trades Filed: {data['new_trades_count']}",
+            trades_label,
            f"  • Market Alerts: {data['market_alerts_count']}",
            f"  • Critical Alerts (≥7 severity): {data['critical_alerts_count']}",
            f"  • Suspicious Timing Trades: {data['suspicious_trades_count']}",
@@ -287,6 +305,11 @@ class ReportGenerator:
    def _format_daily_html(self, data: Dict[str, Any]) -> str:
        """Format daily report as HTML."""
        if data.get("lookback_days", 1) > 1:
            new_trades_label = f"Trades Filed (last {data['lookback_days']} days):"
        else:
            new_trades_label = "New Trades:"
        html = f"""
        <html>
        <head>
@@ -307,7 +330,7 @@ class ReportGenerator:
            <div class="summary">
                <h2>📊 Summary</h2>
-                <div class="stat"><strong>New Trades:</strong> {data['new_trades_count']}</div>
+                <div class="stat"><strong>{new_trades_label}</strong> {data['new_trades_count']}</div>
                <div class="stat"><strong>Market Alerts:</strong> {data['market_alerts_count']}</div>
                <div class="stat"><strong>Critical Alerts:</strong> {data['critical_alerts_count']}</div>
                <div class="stat"><strong>Suspicious Trades:</strong> {data['suspicious_trades_count']}</div>
Author	SHA1	Message	Date
ilia	023177d77d	ci: allowlist known historical/fixture gitleaks findings CI / skip-ci-check (pull_request) Successful in 12s Details CI / secret-scan (pull_request) Successful in 11s Details CI / python-ci (pull_request) Successful in 26s Details	2026-07-13 15:55:40 -04:00
ilia	0017de6fec	ci: fix gitleaks no-op ruleset (add useDefault = true) CI / skip-ci-check (pull_request) Successful in 10s Details CI / secret-scan (pull_request) Failing after 10s Details CI / python-ci (pull_request) Successful in 29s Details	2026-07-12 18:15:24 -05:00
ilia	c991db09e9	ci: add local pre-commit gitleaks hook CI / skip-ci-check (pull_request) Successful in 11s Details CI / secret-scan (pull_request) Successful in 8s Details CI / python-ci (pull_request) Successful in 22s Details	2026-07-12 16:21:28 -05:00
ilia	11bd00893e	ci: add local pre-commit gitleaks hook	2026-07-12 16:21:28 -05:00
ilia	781a45f1a3	Mark deploy and daily email verified (#4 ) CI / skip-ci-check (push) Successful in 9s Details CI / secret-scan (push) Successful in 8s Details CI / python-ci (push) Successful in 22s Details	2026-07-12 10:38:51 -05:00
ilia	5d1fc601ea	Refresh handoff and harden daily report (#3 ) CI / skip-ci-check (push) Successful in 17s Details CI / secret-scan (push) Successful in 8s Details CI / python-ci (push) Successful in 27s Details	2026-07-12 10:26:17 -05:00
ilia	5c0385e27c	Merge pull request 'Add homelab Gitea Actions CI (python-scraper)' (#2 ) from ci/bootstrap-gitea-actions-python into main CI / skip-ci-check (push) Successful in 10s Details CI / secret-scan (push) Successful in 8s Details CI / python-ci (push) Successful in 22s Details	2026-05-29 21:47:12 -05:00
ilia	8110c5949d	ci: sync gitleaks allowlist CI / skip-ci-check (pull_request) Successful in 10s Details CI / secret-scan (pull_request) Successful in 9s Details CI / python-ci (pull_request) Successful in 31s Details	2026-05-29 21:31:14 -05:00
ilia	fc510f2b2c	ci: refresh workflow (re-run pipelines)	2026-05-29 21:31:13 -05:00
ilia	9cb05ddf77	ci: sync gitleaks allowlist CI / skip-ci-check (pull_request) Successful in 11s Details CI / secret-scan (pull_request) Successful in 10s Details CI / python-ci (pull_request) Successful in 35s Details	2026-05-29 21:27:44 -05:00
ilia	1f4e9c075a	ci: refresh workflow (re-run pipelines) CI / skip-ci-check (pull_request) Successful in 11s Details CI / secret-scan (pull_request) Successful in 10s Details CI / python-ci (pull_request) Successful in 34s Details	2026-05-29 21:27:42 -05:00
ilia	a95429509f	ci: sync gitleaks allowlist CI / skip-ci-check (pull_request) Successful in 11s Details CI / secret-scan (pull_request) Successful in 12s Details CI / python-ci (pull_request) Successful in 29s Details	2026-05-29 21:23:20 -05:00
ilia	169f28363b	ci: refresh workflow (re-run pipelines)	2026-05-29 21:23:18 -05:00
ilia	b383f9dd8d	ci: add homelab gitleaks allowlist CI / skip-ci-check (pull_request) Successful in 13s Details CI / secret-scan (pull_request) Successful in 9s Details CI / python-ci (pull_request) Successful in 31s Details	2026-05-29 21:18:58 -05:00
ilia	a8757fd6f1	ci: refresh workflow (re-run pipelines)	2026-05-29 21:18:54 -05:00
ilia	3950867dae	ci: refresh workflow (re-run pipelines) CI / skip-ci-check (pull_request) Successful in 10s Details CI / secret-scan (pull_request) Failing after 9s Details CI / python-ci (pull_request) Successful in 34s Details	2026-05-29 20:29:44 -05:00
ilia	1bce7581e5	ci: refresh workflow (re-run pipelines) CI / skip-ci-check (pull_request) Successful in 10s Details CI / python-ci (pull_request) Failing after 10s Details CI / secret-scan (pull_request) Failing after 9s Details	2026-05-29 20:27:19 -05:00
ilia	b9a2e1011f	ci: sync workflow template CI / skip-ci-check (pull_request) Successful in 10s Details CI / python-ci (pull_request) Failing after 9s Details CI / secret-scan (pull_request) Failing after 10s Details	2026-05-29 20:25:06 -05:00
ilia	a11108838d	ci: refresh workflow (re-run pipelines) CI / skip-ci-check (pull_request) Successful in 9s Details CI / python-ci (pull_request) Failing after 9s Details CI / secret-scan (pull_request) Failing after 9s Details	2026-05-29 20:19:46 -05:00
ilia	2ee601c198	ci: sync workflow template (node container + host fixes) CI / skip-ci-check (pull_request) Successful in 10s Details CI / python-ci (pull_request) Failing after 14s Details CI / secret-scan (pull_request) Failing after 10s Details	2026-05-29 20:14:05 -05:00
ilia	367d76eb9d	ci: add homelab Gitea Actions workflow (ci-python.yml) CI / skip-ci-check (pull_request) Failing after 8s Details CI / python-ci (pull_request) Has been skipped Details CI / secret-scan (pull_request) Has been skipped Details	2026-05-29 16:14:56 -05:00
ilia	274ad5b5d5	Merge pull request 'Fix live ingest, email config, and dependencies for homelab deploy.' (#1 ) from deploy/homelab-lxc236 into main CI / security-scan (push) Successful in 43s Details CI / lint-and-test (push) Successful in 54s Details CI / dependency-scan (push) Successful in 13s Details CI / docker-build-test (push) Successful in 7s Details CI / workflow-summary (push) Successful in 4s Details Reviewed-on: #1	2026-05-26 19:20:59 -05:00
ilia	bc68f8a752	Use project venv in CI to satisfy PEP 668 on Ubuntu runner. CI / lint-and-test (pull_request) Successful in 59s Details CI / security-scan (pull_request) Successful in 51s Details CI / docker-build-test (pull_request) Successful in 7s Details CI / dependency-scan (pull_request) Successful in 14s Details CI / workflow-summary (pull_request) Successful in 4s Details Gitea act_runner images block system-wide pip; install deps into .venv and invoke tools from .venv/bin.	2026-05-26 20:17:56 -04:00
ilia	648d5ac742	Fix CI for Gitea act_runner: python3 and local docker build. CI / lint-and-test (pull_request) Failing after 7s Details CI / security-scan (pull_request) Failing after 6s Details CI / dependency-scan (pull_request) Successful in 13s Details CI / docker-build-test (pull_request) Successful in 45s Details CI / workflow-summary (pull_request) Successful in 4s Details Use python3 instead of unavailable python3.11 packages, drop apt step that failed on runner image, and replace buildx with docker build so the test step can run the built image.	2026-05-26 20:07:49 -04:00
ilia	8dd354c7c5	Fix Gitea Actions CI for act_runner on homelab. CI / security-scan (pull_request) Failing after 6s Details CI / lint-and-test (pull_request) Failing after 10s Details CI / dependency-scan (pull_request) Successful in 13s Details CI / docker-build-test (pull_request) Failing after 1m4s Details CI / workflow-summary (pull_request) Successful in 3s Details Remove Python job containers so checkout@v4 can run (needs Node), install deps on ubuntu-latest for security-scan, and stop excluding README.md from Docker build context.	2026-05-26 20:00:48 -04:00
ilia	31c656d66f	Fix live ingest, email config, and dependencies for homelab deploy. CI / security-scan (pull_request) Failing after 28s Details CI / lint-and-test (pull_request) Failing after 31s Details CI / dependency-scan (pull_request) Successful in 22s Details CI / docker-build-test (pull_request) Failing after 34s Details CI / workflow-summary (pull_request) Successful in 4s Details Switch HouseWatcherClient to public STOCK Act JSON (legacy housestockwatcher.com is down), add httpx/scikit-learn to pyproject, wire SMTP settings through Settings, fix get_session() as a context manager, and use savepoints in TradeLoader so one bad row does not roll back the batch.	2026-05-26 19:52:29 -04:00