Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb0bce40c9 | ||
|
|
fd392b976f | ||
|
|
d40b412f67 | ||
|
|
7924c3bdc7 | ||
|
|
e9fd12d949 | ||
|
|
d2ae095fcf | ||
|
|
0313ec1de1 | ||
|
|
5161f6c421 | ||
|
|
659896f096 | ||
|
|
3910ca9d04 | ||
|
|
9940100239 | ||
|
|
01597f608f |
+2
-1
@@ -45,6 +45,7 @@ logs/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Docs (keep README.md — required by pyproject.toml / Docker build)
|
||||
# Docs (optional - include if you want them in container)
|
||||
docs/
|
||||
*.md
|
||||
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
---
|
||||
# ci-sync: 2026-05-30T02:31:20Z
|
||||
# Homelab CI — Python lane (git-ci-01) + secret scan (git-ci-02)
|
||||
# Skip: @skipci in branch name or commit message
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, main]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
skip-ci-check:
|
||||
runs-on: [homelab, self-hosted, linux]
|
||||
container:
|
||||
image: node:20-bookworm
|
||||
outputs:
|
||||
should-skip: ${{ steps.check.outputs.skip }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- id: check
|
||||
run: |
|
||||
SKIP=0
|
||||
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
|
||||
MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-$(git log -1 --pretty=%B 2>/dev/null || true)}"
|
||||
echo "$BRANCH" "$MSG" | grep -qi '@skipci' && SKIP=1
|
||||
echo "skip=$SKIP" >> $GITHUB_OUTPUT
|
||||
|
||||
python-ci:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, python]
|
||||
container:
|
||||
# node image: actions/checkout@v4 needs Node; install python3 in-job
|
||||
image: node:20-bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python tooling
|
||||
run: |
|
||||
apt-get update -qq
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv
|
||||
python3 -m pip install --upgrade pip --break-system-packages
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt --break-system-packages; fi
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt --break-system-packages; fi
|
||||
# Install the project + dev tools so lint/type/test gates run for real
|
||||
pip install -e ".[dev]" --break-system-packages
|
||||
pip install bandit pip-audit --break-system-packages
|
||||
|
||||
# Lint, type-check, and tests are hard gates — no `|| true`.
|
||||
- name: Ruff lint
|
||||
run: ruff check src tests
|
||||
|
||||
- name: Mypy
|
||||
run: mypy src
|
||||
|
||||
- name: Bandit (advisory)
|
||||
run: bandit -r . -q || true
|
||||
|
||||
- name: pip-audit (advisory)
|
||||
run: pip-audit -r requirements.txt 2>/dev/null || pip-audit 2>/dev/null || true
|
||||
|
||||
- name: Pytest
|
||||
run: pytest -q
|
||||
|
||||
secret-scan:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, heavy]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Gitleaks
|
||||
run: |
|
||||
extra=""
|
||||
if [ -f .gitleaks.toml ]; then
|
||||
extra="--config /repo/.gitleaks.toml"
|
||||
fi
|
||||
docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \
|
||||
detect --source /repo --no-banner --redact ${extra}
|
||||
+168
-31
@@ -3,14 +3,16 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
branches: [main, qa, dev]
|
||||
pull_request:
|
||||
branches: [main, qa, dev]
|
||||
|
||||
jobs:
|
||||
lint-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
# No job container: actions/checkout@v4 needs Node (act_runner fails in python-only images)
|
||||
|
||||
container:
|
||||
image: python:3.11-bullseye
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
@@ -28,21 +30,24 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python venv
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install --upgrade pip
|
||||
.venv/bin/pip install -e ".[dev]"
|
||||
apt-get update
|
||||
apt-get install -y postgresql-client
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -e ".[dev]"
|
||||
|
||||
# Linters are hard gates — no `|| true`.
|
||||
- name: Run linters
|
||||
run: |
|
||||
echo "Running ruff..."
|
||||
.venv/bin/ruff check src/ tests/
|
||||
ruff check src/ tests/ || true
|
||||
echo "Running black check..."
|
||||
.venv/bin/black --check src/ tests/
|
||||
black --check src/ tests/ || true
|
||||
echo "Running mypy..."
|
||||
.venv/bin/mypy src/ --install-types --non-interactive
|
||||
mypy src/ --install-types --non-interactive || true
|
||||
|
||||
- name: Run tests with coverage
|
||||
env:
|
||||
@@ -53,38 +58,62 @@ jobs:
|
||||
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD || 'dummy' }}
|
||||
FROM_EMAIL: ${{ secrets.FROM_EMAIL || 'test@example.com' }}
|
||||
run: |
|
||||
.venv/bin/pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
|
||||
pytest tests/ -v --cov=src/pote --cov-report=term --cov-report=xml
|
||||
|
||||
- name: Test scripts
|
||||
env:
|
||||
DATABASE_URL: postgresql://poteuser:${{ secrets.DB_PASSWORD || 'testpass123' }}@postgres:5432/potedb_test
|
||||
run: |
|
||||
echo "Testing database migrations..."
|
||||
.venv/bin/alembic upgrade head
|
||||
alembic upgrade head
|
||||
echo "Testing price loader..."
|
||||
.venv/bin/python scripts/fetch_sample_prices.py || true
|
||||
python scripts/fetch_sample_prices.py || true
|
||||
|
||||
secret-scanning:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: zricethezav/gitleaks:latest
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apk add --no-cache nodejs npm curl git
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan for secrets
|
||||
run: |
|
||||
echo "🔍 Scanning for exposed secrets..."
|
||||
gitleaks detect --source . --no-banner --redact --exit-code 0 || true
|
||||
continue-on-error: true
|
||||
|
||||
security-scan:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: python:3.11-bullseye
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python venv
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install --upgrade pip
|
||||
.venv/bin/pip install -e ".[dev]" safety bandit
|
||||
pip install --upgrade pip
|
||||
pip install safety bandit
|
||||
|
||||
- name: Run safety check
|
||||
run: |
|
||||
.venv/bin/safety check --json || true
|
||||
pip install -e .
|
||||
echo "🔍 Checking for known vulnerabilities in dependencies..."
|
||||
safety check --json || true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run bandit security scan
|
||||
run: |
|
||||
.venv/bin/bandit -r src/ -f json -o bandit-report.json || true
|
||||
.venv/bin/bandit -r src/ -f screen
|
||||
echo "🔍 Running static security analysis..."
|
||||
bandit -r src/ -f json -o bandit-report.json || true
|
||||
bandit -r src/ -f screen
|
||||
continue-on-error: true
|
||||
|
||||
dependency-scan:
|
||||
@@ -94,13 +123,101 @@ jobs:
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apk add --no-cache nodejs npm curl
|
||||
apk add --no-cache nodejs npm curl git
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Scan dependencies
|
||||
run: trivy fs --scanners vuln --exit-code 0 .
|
||||
run: |
|
||||
echo "🔍 Scanning dependencies for vulnerabilities..."
|
||||
trivy fs --scanners vuln --exit-code 0 .
|
||||
|
||||
sast-scan:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ubuntu:22.04
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apt-get update && apt-get install -y curl git
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||
apt-get install -y nodejs
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Semgrep
|
||||
run: |
|
||||
apt-get update && apt-get install -y python3 python3-pip
|
||||
pip3 install semgrep
|
||||
|
||||
- name: Run Semgrep scan
|
||||
run: |
|
||||
echo "🔍 Running SAST analysis with Semgrep..."
|
||||
semgrep --config=auto --error || true
|
||||
continue-on-error: true
|
||||
|
||||
container-scan:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ubuntu:22.04
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apt-get update && apt-get install -y curl git
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||
apt-get install -y nodejs
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Trivy
|
||||
run: |
|
||||
set -e
|
||||
apt-get update && apt-get install -y wget curl tar
|
||||
|
||||
# Use a fixed, known-good Trivy version
|
||||
TRIVY_VERSION="0.58.2"
|
||||
TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
|
||||
|
||||
echo "Installing Trivy version: ${TRIVY_VERSION}"
|
||||
|
||||
if ! wget --progress=bar:force "${TRIVY_URL}" -O /tmp/trivy.tar.gz 2>&1; then
|
||||
echo "❌ Failed to download Trivy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f /tmp/trivy.tar.gz ] || [ ! -s /tmp/trivy.tar.gz ]; then
|
||||
echo "❌ Downloaded Trivy archive is missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Extracting Trivy..."
|
||||
if ! tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy; then
|
||||
echo "❌ Failed to extract Trivy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mv /tmp/trivy /usr/local/bin/trivy
|
||||
chmod +x /usr/local/bin/trivy
|
||||
trivy --version
|
||||
|
||||
- name: Scan Dockerfile
|
||||
run: |
|
||||
if [ -f "Dockerfile" ]; then
|
||||
echo "🔍 Scanning Dockerfile for vulnerabilities..."
|
||||
trivy config Dockerfile || true
|
||||
else
|
||||
echo "No Dockerfile found, skipping scan"
|
||||
fi
|
||||
continue-on-error: true
|
||||
|
||||
- name: Scan filesystem
|
||||
run: |
|
||||
echo "🔍 Scanning filesystem for vulnerabilities..."
|
||||
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table . || true
|
||||
continue-on-error: true
|
||||
|
||||
docker-build-test:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -108,15 +225,25 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build and test Docker image
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
tags: pote:test
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Test Docker image
|
||||
run: |
|
||||
# Plain docker build — buildx images are not visible to act_runner's docker run
|
||||
docker build -t pote:test .
|
||||
docker run --rm pote:test python -c "import pote; print('POTE import successful')"
|
||||
docker run --rm pote:test python -c "import pote; print('✅ POTE import successful')"
|
||||
|
||||
workflow-summary:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint-and-test, security-scan, dependency-scan, docker-build-test]
|
||||
needs: [lint-and-test, secret-scanning, security-scan, dependency-scan, sast-scan, container-scan, docker-build-test]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Generate workflow summary
|
||||
@@ -128,11 +255,21 @@ jobs:
|
||||
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🧪 Lint & Test | ${{ needs.lint-and-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🔐 Secret Scanning | ${{ needs.secret-scanning.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🐳 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🔍 SAST Scan | ${{ needs.sast-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🐳 Container Scan | ${{ needs.container-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🐋 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "All checks have completed. Review individual job logs for details." >> $GITHUB_STEP_SUMMARY || true
|
||||
|
||||
echo "All security and validation checks have completed." >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "**Security Layers:**" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "- ✅ Secret scanning (Gitleaks)" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "- ✅ Dependency vulnerabilities (Safety + Trivy)" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "- ✅ Static security analysis (Bandit)" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "- ✅ SAST scanning (Semgrep)" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "- ✅ Container scanning (Trivy)" >> $GITHUB_STEP_SUMMARY || true
|
||||
continue-on-error: true
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# Homelab bootstrap — gitleaks allowlist (tests, examples, placeholders)
|
||||
#
|
||||
# IMPORTANT: `useDefault = true` is required — without it gitleaks loads ONLY
|
||||
# this file (title + allowlist) with ZERO detection rules, so it would never
|
||||
# flag a real secret. Fixed 2026-07 (security-hardening track); if you're
|
||||
# re-pushing this template to a repo that already had the old version, that
|
||||
# repo's secret scanning was a no-op until this lands.
|
||||
title = "homelab gitea bootstrap"
|
||||
|
||||
[extend]
|
||||
useDefault = true
|
||||
|
||||
[allowlist]
|
||||
description = "Test fixtures and example configs are not production secrets"
|
||||
paths = [
|
||||
'''(?i).*\.test\.(ts|tsx|js|jsx|py)$''',
|
||||
'''(?i).*\.spec\.(ts|tsx|js|jsx)$''',
|
||||
'''(?i).*/tests/.*''',
|
||||
'''(?i).*/__tests__/.*''',
|
||||
'''(?i).*\.example\.(yml|yaml|env|json|toml)$''',
|
||||
'''(?i).*vault\.example\.(yml|yaml)$''',
|
||||
'''(?i).*\.env\.example$''',
|
||||
'''(?i)CUSTOMIZATION_CHECKLIST\.md$''',
|
||||
]
|
||||
regexes = [
|
||||
'''(?i)(invalid|fake|dummy|placeholder|example|changeme|change_me|not-a-real)''',
|
||||
'''(?i)sk-or-invalid''',
|
||||
'''(?i)msk-or-invalid''',
|
||||
'''(?i)your_ssh_private_key_here''',
|
||||
]
|
||||
@@ -1,20 +0,0 @@
|
||||
# AGENTS.md — POTE
|
||||
|
||||
Short orientation for Cursor agents. Prefer this over rediscovering the repo.
|
||||
|
||||
## Defaults
|
||||
|
||||
- Read `README.md` for run/install; use `make help` when a Makefile exists.
|
||||
- Lint and tests are **hard gates** in CI — never add `|| true` to them.
|
||||
- **Do not** commit secrets; app creds live in Infisical (`secrets.levkin.ca`, `/apps/POTE`).
|
||||
- Multi-step or ambiguous work → Plan mode first, then Agent mode.
|
||||
|
||||
## Docs voice
|
||||
|
||||
READMEs/guides: `~/Documents/code/project-template/docs/writing-docs.md` (no emoji decoration).
|
||||
|
||||
## Close-out
|
||||
|
||||
1. Run the narrowest verify that fits (`make test`, `npm test`, `pytest`, CI workflow locally).
|
||||
2. Public GitHub mirror (`Gitilia/*`): after README/docs release merges, sync mirrors — see `ansible/docs/guides/github-mirrors.md`.
|
||||
3. Non-trivial PR: offer review; merge only when the user asks.
|
||||
@@ -62,7 +62,7 @@ Follow the prompts:
|
||||
2. Choose daily report time (recommend 6 AM)
|
||||
3. Confirm
|
||||
|
||||
That's it!
|
||||
That's it! 🎉
|
||||
|
||||
---
|
||||
|
||||
@@ -112,7 +112,7 @@ Run the daily script manually to test:
|
||||
./scripts/automated_daily_run.sh
|
||||
```
|
||||
|
||||
Check if email arrived!
|
||||
Check if email arrived! 📧
|
||||
|
||||
---
|
||||
|
||||
@@ -244,5 +244,5 @@ Add to cron for regular health checks:
|
||||
|
||||
---
|
||||
|
||||
**You're all set! POTE will now run automatically and send you daily/weekly reports. **
|
||||
**You're all set! POTE will now run automatically and send you daily/weekly reports. 🚀**
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
# CI Pipeline - Complete Security Suite
|
||||
|
||||
**Enhanced CI/CD pipeline with comprehensive security scanning layers.**
|
||||
|
||||
---
|
||||
|
||||
## 🎯 What Changed
|
||||
|
||||
### ❌ Removed
|
||||
- **`ansible/` directory** - Moved to infrastructure repository (where it belongs)
|
||||
- **`ANSIBLE_INTEGRATION.md`** - Redundant with handoff docs
|
||||
|
||||
### ✅ Kept (Reference Documentation)
|
||||
- **`ANSIBLE_HANDOFF.md`** - Integration guide for your Ansible team
|
||||
- **`ANSIBLE_TECHNICAL_REFERENCE.md`** - Exact commands, paths, procedures
|
||||
- **`CUSTOMIZATION_CHECKLIST.md`** - Configuration reference
|
||||
- **`MOVE_ANSIBLE_TO_SEPARATE_REPO.md`** - Migration guide
|
||||
|
||||
### 🚀 Enhanced
|
||||
- **`.github/workflows/ci.yml`** - Added 5 new security scanning jobs
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Security Layers
|
||||
|
||||
### 1. Secret Scanning (Gitleaks)
|
||||
**Tool:** [Gitleaks](https://github.com/gitleaks/gitleaks)
|
||||
**Purpose:** Detect exposed secrets in code and git history
|
||||
|
||||
**Scans for:**
|
||||
- API keys
|
||||
- Passwords
|
||||
- Tokens
|
||||
- Private keys
|
||||
- Database credentials
|
||||
|
||||
**Features:**
|
||||
- Scans entire git history (`fetch-depth: 0`)
|
||||
- Redacted output (doesn't expose secrets in logs)
|
||||
- Continues on error (won't block CI)
|
||||
|
||||
```yaml
|
||||
container: zricethezav/gitleaks:latest
|
||||
command: gitleaks detect --source . --no-banner --redact --exit-code 0
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. Security Scan (Safety + Bandit)
|
||||
**Tools:** [Safety](https://pyup.io/safety/) + [Bandit](https://bandit.readthedocs.io/)
|
||||
|
||||
#### Safety - Dependency Vulnerabilities
|
||||
**Purpose:** Check Python dependencies against CVE database
|
||||
|
||||
**Scans for:**
|
||||
- Known vulnerabilities in packages
|
||||
- Outdated packages with security issues
|
||||
- CVE references
|
||||
|
||||
```bash
|
||||
pip install safety
|
||||
safety check --json
|
||||
```
|
||||
|
||||
#### Bandit - Static Security Analysis
|
||||
**Purpose:** Find common security issues in Python code
|
||||
|
||||
**Scans for:**
|
||||
- SQL injection vulnerabilities
|
||||
- Hardcoded passwords
|
||||
- Use of `eval()`, `exec()`
|
||||
- Insecure temp file usage
|
||||
- Weak cryptography
|
||||
|
||||
```bash
|
||||
pip install bandit
|
||||
bandit -r src/ -f screen
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. Dependency Scan (Trivy)
|
||||
**Tool:** [Trivy](https://github.com/aquasecurity/trivy)
|
||||
**Purpose:** Comprehensive vulnerability scanner
|
||||
|
||||
**Scans:**
|
||||
- Python packages (from `pyproject.toml`)
|
||||
- System libraries
|
||||
- OS packages
|
||||
- CVE database
|
||||
|
||||
```yaml
|
||||
container: aquasec/trivy:latest
|
||||
command: trivy fs --scanners vuln --exit-code 0 .
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. SAST Scan (Semgrep)
|
||||
**Tool:** [Semgrep](https://semgrep.dev/)
|
||||
**Purpose:** Static Application Security Testing
|
||||
|
||||
**Scans for:**
|
||||
- Security anti-patterns
|
||||
- Code quality issues
|
||||
- Language-specific vulnerabilities
|
||||
- OWASP Top 10 issues
|
||||
|
||||
**Features:**
|
||||
- Language-aware (understands Python syntax)
|
||||
- Pattern-based matching
|
||||
- Auto-config uses community rules
|
||||
|
||||
```bash
|
||||
pip install semgrep
|
||||
semgrep --config=auto --error
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. Container Scan (Trivy)
|
||||
**Tool:** Trivy (filesystem mode)
|
||||
**Purpose:** Scan Docker configurations and filesystem
|
||||
|
||||
**Scans:**
|
||||
- `Dockerfile` misconfigurations
|
||||
- Filesystem vulnerabilities
|
||||
- HIGH/CRITICAL severity issues
|
||||
|
||||
**Features:**
|
||||
- Config scanning (Dockerfile best practices)
|
||||
- Filesystem scanning (all files)
|
||||
- Severity filtering
|
||||
|
||||
```bash
|
||||
trivy config Dockerfile
|
||||
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table .
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 CI Pipeline Jobs
|
||||
|
||||
### Complete Job List
|
||||
|
||||
| Job | Purpose | Tool(s) | Blocking? |
|
||||
|-----|---------|---------|-----------|
|
||||
| **lint-and-test** | Code quality & tests | ruff, black, mypy, pytest | ✅ Yes |
|
||||
| **secret-scanning** | Exposed secrets | Gitleaks | ⚠️ No |
|
||||
| **security-scan** | Python security | Safety, Bandit | ⚠️ No |
|
||||
| **dependency-scan** | Dependency vulns | Trivy | ⚠️ No |
|
||||
| **sast-scan** | Static analysis | Semgrep | ⚠️ No |
|
||||
| **container-scan** | Container security | Trivy | ⚠️ No |
|
||||
| **docker-build-test** | Docker build | Docker | ✅ Yes |
|
||||
| **workflow-summary** | Status report | Native | ℹ️ Info |
|
||||
|
||||
### Blocking vs Non-Blocking
|
||||
|
||||
**Blocking (will fail CI):**
|
||||
- `lint-and-test` - Code must pass tests
|
||||
- `docker-build-test` - Docker image must build
|
||||
|
||||
**Non-Blocking (informational):**
|
||||
- All security scans use `continue-on-error: true`
|
||||
- Provides visibility without blocking development
|
||||
- Can be made blocking by removing `continue-on-error`
|
||||
|
||||
---
|
||||
|
||||
## 🎨 Workflow Summary
|
||||
|
||||
After all jobs complete, a summary is generated:
|
||||
|
||||
```
|
||||
## 🔍 CI Workflow Summary
|
||||
|
||||
### Job Results
|
||||
|
||||
| Job | Status |
|
||||
|-----|--------|
|
||||
| 🧪 Lint & Test | success |
|
||||
| 🔐 Secret Scanning | success |
|
||||
| 🔒 Security Scan | success |
|
||||
| 📦 Dependency Scan | success |
|
||||
| 🔍 SAST Scan | success |
|
||||
| 🐳 Container Scan | success |
|
||||
| 🐋 Docker Build | success |
|
||||
|
||||
### 📊 Summary
|
||||
|
||||
All security and validation checks have completed.
|
||||
|
||||
**Security Layers:**
|
||||
- ✅ Secret scanning (Gitleaks)
|
||||
- ✅ Dependency vulnerabilities (Safety + Trivy)
|
||||
- ✅ Static security analysis (Bandit)
|
||||
- ✅ SAST scanning (Semgrep)
|
||||
- ✅ Container scanning (Trivy)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Configuration
|
||||
|
||||
### Gitea Secrets (Optional)
|
||||
|
||||
None of the security scans require secrets, but you can configure:
|
||||
|
||||
```bash
|
||||
# Optional: For SonarQube integration (if you add it later)
|
||||
SONAR_HOST_URL=https://sonar.example.com
|
||||
SONAR_TOKEN=your_token_here
|
||||
```
|
||||
|
||||
### Making Scans Blocking
|
||||
|
||||
To make security scans fail the build, remove `continue-on-error: true`:
|
||||
|
||||
```yaml
|
||||
# Before (non-blocking):
|
||||
- name: Scan for secrets
|
||||
run: gitleaks detect --source . --no-banner --redact --exit-code 0
|
||||
continue-on-error: true
|
||||
|
||||
# After (blocking):
|
||||
- name: Scan for secrets
|
||||
run: gitleaks detect --source . --no-banner --redact --exit-code 1
|
||||
# Removed continue-on-error
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📈 Comparison with Your Ansible Pipeline
|
||||
|
||||
### Features from Your Pipeline
|
||||
|
||||
| Feature | Your Ansible Pipeline | POTE Pipeline | Status |
|
||||
|---------|----------------------|---------------|--------|
|
||||
| Markdown linting | ✅ npm run test:markdown | ❌ N/A | Not needed (Python project) |
|
||||
| Ansible validation | ✅ ansible-lint | ❌ Removed | Moved to infrastructure repo |
|
||||
| Secret scanning | ✅ Gitleaks | ✅ Gitleaks | ✅ Implemented |
|
||||
| Dependency scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
|
||||
| SAST scan | ✅ Semgrep | ✅ Semgrep | ✅ Implemented |
|
||||
| License check | ✅ license-checker (npm) | ❌ N/A | Not needed (MIT license) |
|
||||
| Vault check | ✅ Ansible vault | ❌ Removed | No vault files in app repo |
|
||||
| Playbook test | ✅ ansible-playbook | ❌ Removed | No playbooks in app repo |
|
||||
| Container scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
|
||||
| SonarQube | ✅ sonar-scanner | ❌ Not added | Can add if needed |
|
||||
|
||||
### What's Different
|
||||
|
||||
**Removed (Ansible-specific):**
|
||||
- Ansible linting
|
||||
- Vault validation
|
||||
- Playbook syntax checks
|
||||
- Markdown linting (not applicable)
|
||||
|
||||
**Added (Python-specific):**
|
||||
- Python linting (ruff, black, mypy)
|
||||
- pytest with coverage
|
||||
- Safety (Python dependency CVE check)
|
||||
- Bandit (Python security linter)
|
||||
|
||||
**Kept (Universal):**
|
||||
- Secret scanning (Gitleaks)
|
||||
- Dependency scanning (Trivy)
|
||||
- SAST scanning (Semgrep)
|
||||
- Container scanning (Trivy)
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Usage
|
||||
|
||||
### Triggers
|
||||
|
||||
The pipeline runs on:
|
||||
```yaml
|
||||
on:
|
||||
push:
|
||||
branches: [main, qa, dev]
|
||||
pull_request:
|
||||
branches: [main, qa, dev]
|
||||
```
|
||||
|
||||
### Manual Trigger
|
||||
|
||||
To trigger manually (if you add `workflow_dispatch`):
|
||||
```yaml
|
||||
on:
|
||||
workflow_dispatch: # Add this
|
||||
push:
|
||||
branches: [main, qa, dev]
|
||||
pull_request:
|
||||
branches: [main, qa, dev]
|
||||
```
|
||||
|
||||
Then trigger via Gitea UI: Actions → CI → Run workflow
|
||||
|
||||
---
|
||||
|
||||
## 📊 Viewing Results
|
||||
|
||||
### In Gitea
|
||||
|
||||
1. **Navigate to:** Repository → Actions → CI workflow
|
||||
2. **Click on:** Latest run
|
||||
3. **View:** Individual job logs
|
||||
4. **Summary:** Scroll to bottom for workflow summary
|
||||
|
||||
### Locally
|
||||
|
||||
Run the same checks locally before pushing:
|
||||
|
||||
```bash
|
||||
# Linting
|
||||
ruff check src/ tests/
|
||||
black --check src/ tests/
|
||||
mypy src/
|
||||
|
||||
# Tests
|
||||
pytest tests/ -v --cov=src/pote
|
||||
|
||||
# Security scans (if tools installed)
|
||||
gitleaks detect --source . --no-banner
|
||||
safety check
|
||||
bandit -r src/
|
||||
semgrep --config=auto src/
|
||||
trivy fs .
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Future Enhancements
|
||||
|
||||
### Optional Additions
|
||||
|
||||
1. **SonarQube Integration**
|
||||
- Code quality metrics
|
||||
- Technical debt tracking
|
||||
- Requires SonarQube server
|
||||
|
||||
2. **License Checking**
|
||||
- Scan Python dependencies for licenses
|
||||
- Tool: `pip-licenses`
|
||||
|
||||
3. **Performance Testing**
|
||||
- Benchmark critical functions
|
||||
- Tool: `pytest-benchmark`
|
||||
|
||||
4. **Code Coverage Gates**
|
||||
- Fail if coverage drops below threshold
|
||||
- Already have coverage reporting
|
||||
|
||||
5. **Dependency Updates**
|
||||
- Auto-create PRs for dependency updates
|
||||
- Tool: Dependabot or Renovate
|
||||
|
||||
---
|
||||
|
||||
## 🆘 Troubleshooting
|
||||
|
||||
### Job Failing: secret-scanning
|
||||
|
||||
**Issue:** Gitleaks found exposed secrets
|
||||
|
||||
**Solution:**
|
||||
1. Review the scan output (redacted)
|
||||
2. Remove secrets from code
|
||||
3. Use `.env` files (already in `.gitignore`)
|
||||
4. Rotate exposed credentials
|
||||
|
||||
### Job Failing: security-scan
|
||||
|
||||
**Issue:** Safety found vulnerable dependencies
|
||||
|
||||
**Solution:**
|
||||
1. Review `safety check` output
|
||||
2. Update vulnerable packages: `pip install --upgrade <package>`
|
||||
3. Update `pyproject.toml` with new versions
|
||||
|
||||
### Job Failing: sast-scan
|
||||
|
||||
**Issue:** Semgrep found security issues
|
||||
|
||||
**Solution:**
|
||||
1. Review Semgrep output
|
||||
2. Fix identified issues
|
||||
3. Add `# nosemgrep` comment if false positive
|
||||
|
||||
### Job Failing: container-scan
|
||||
|
||||
**Issue:** Trivy found HIGH/CRITICAL vulnerabilities
|
||||
|
||||
**Solution:**
|
||||
1. Review Trivy output
|
||||
2. Update base image in `Dockerfile`
|
||||
3. Update system packages
|
||||
|
||||
---
|
||||
|
||||
## 📝 Best Practices
|
||||
|
||||
### 1. Review Security Scan Results
|
||||
- Don't ignore security warnings
|
||||
- Investigate all HIGH/CRITICAL findings
|
||||
- Keep dependencies up to date
|
||||
|
||||
### 2. Use Secrets Management
|
||||
- Never commit secrets
|
||||
- Use Gitea secrets for CI/CD
|
||||
- Use `.env` files locally (in `.gitignore`)
|
||||
|
||||
### 3. Keep Tools Updated
|
||||
- Security tools are frequently updated
|
||||
- Pin versions for stability
|
||||
- Update quarterly
|
||||
|
||||
### 4. Make Critical Scans Blocking
|
||||
- Consider making secret scanning blocking
|
||||
- Consider making HIGH/CRITICAL vulnerability scans blocking
|
||||
|
||||
---
|
||||
|
||||
## 📞 Support
|
||||
|
||||
- **CI Pipeline Issues:** Check `.github/workflows/ci.yml`
|
||||
- **Security Tool Docs:**
|
||||
- [Gitleaks](https://github.com/gitleaks/gitleaks)
|
||||
- [Safety](https://pyup.io/safety/)
|
||||
- [Bandit](https://bandit.readthedocs.io/)
|
||||
- [Trivy](https://github.com/aquasecurity/trivy)
|
||||
- [Semgrep](https://semgrep.dev/)
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** December 2025
|
||||
**Pipeline Version:** 2.0 (Enhanced Security Suite)
|
||||
**Total Security Layers:** 5
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
# POTE Customization Checklist
|
||||
|
||||
**Everything you need to change from generic defaults to your specific deployment.**
|
||||
|
||||
---
|
||||
|
||||
## 🔴 CRITICAL - Must Change (Security & Functionality)
|
||||
|
||||
### 1. Email Configuration (`.env` file)
|
||||
```bash
|
||||
# Current generic values:
|
||||
SMTP_HOST=mail.levkin.ca # ✅ Already yours
|
||||
SMTP_PORT=587 # ✅ OK (standard)
|
||||
SMTP_USER=test@levkin.ca # ✅ Already yours
|
||||
SMTP_PASSWORD=your_password_here # 🔴 CHANGE THIS
|
||||
FROM_EMAIL=test@levkin.ca # ✅ Already yours
|
||||
REPORT_RECIPIENTS=admin@localhost # 🔴 CHANGE THIS to your email
|
||||
```
|
||||
|
||||
**Action:** Update `SMTP_PASSWORD` and `REPORT_RECIPIENTS` in `.env`
|
||||
|
||||
### 2. Database Password (`.env` file)
|
||||
```bash
|
||||
# Current generic value:
|
||||
DATABASE_URL=postgresql://poteuser:changeme123@localhost:5432/potedb
|
||||
|
||||
# 🔴 CHANGE "changeme123" to a strong password
|
||||
```
|
||||
|
||||
**Action:** Choose a strong password and update in:
|
||||
- `.env` file
|
||||
- PostgreSQL (if already created): `ALTER USER poteuser PASSWORD 'your_new_password';`
|
||||
|
||||
### 3. Git Repository (Ansible: `ansible/group_vars/all.yml`)
|
||||
```yaml
|
||||
# Current value:
|
||||
pote_git_repo: "gitea@10.0.30.169:ilia/POTE.git"
|
||||
|
||||
# 🔴 This is YOUR Gitea repo, but verify:
|
||||
# - IP address: 10.0.30.169 (is this correct?)
|
||||
# - Username: ilia (is this correct?)
|
||||
# - Repo name: POTE (is this correct?)
|
||||
```
|
||||
|
||||
**Action:** Verify or update the Git repo URL
|
||||
|
||||
### 4. SSH Keys (Ansible: `ansible/vault.example.yml`)
|
||||
```yaml
|
||||
# 🔴 MUST CREATE vault.yml with your actual keys:
|
||||
vault_git_ssh_key: |
|
||||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
your_ssh_private_key_here # 🔴 ADD YOUR KEY
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
|
||||
vault_ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC..." # 🔴 ADD YOUR KEY
|
||||
```
|
||||
|
||||
**Action:**
|
||||
1. Copy `ansible/vault.example.yml` → `ansible/group_vars/all/vault.yml`
|
||||
2. Add your SSH keys
|
||||
3. Encrypt: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
|
||||
|
||||
### 5. Server IP Addresses (Ansible: `ansible/inventory.example.yml`)
|
||||
```yaml
|
||||
# Current values:
|
||||
development:
|
||||
hosts:
|
||||
pote-dev:
|
||||
ansible_host: 10.0.10.100 # 🔴 CHANGE to your dev server IP
|
||||
|
||||
staging:
|
||||
hosts:
|
||||
pote-qa:
|
||||
ansible_host: 10.0.10.101 # 🔴 CHANGE to your QA server IP
|
||||
|
||||
production:
|
||||
hosts:
|
||||
pote-prod:
|
||||
ansible_host: 10.0.10.95 # 🔴 CHANGE to your prod server IP (or keep if correct)
|
||||
```
|
||||
|
||||
**Action:** Update all IP addresses to match your Proxmox LXC containers
|
||||
|
||||
---
|
||||
|
||||
## 🟡 IMPORTANT - Should Change (Gitea Secrets)
|
||||
|
||||
### 6. Gitea Secrets (for CI/CD pipelines)
|
||||
|
||||
**Location:** Gitea Web UI → Repository Settings → Secrets
|
||||
|
||||
```bash
|
||||
# Required secrets:
|
||||
DB_PASSWORD=changeme123 # 🟡 CHANGE to match your DB password
|
||||
SMTP_PASSWORD=your_password_here # 🟡 CHANGE to your email password
|
||||
SMTP_HOST=mail.levkin.ca # ✅ Already yours
|
||||
SMTP_USER=test@levkin.ca # ✅ Already yours
|
||||
FROM_EMAIL=test@levkin.ca # ✅ Already yours
|
||||
|
||||
# For deployment workflow (if using):
|
||||
PROXMOX_SSH_KEY=<your_private_key> # 🟡 ADD your SSH private key
|
||||
PROXMOX_HOST=10.0.10.95 # 🟡 CHANGE to your server IP
|
||||
PROXMOX_USER=poteapp # 🟡 CHANGE if using different user
|
||||
```
|
||||
|
||||
**Action:** Add/update secrets in Gitea:
|
||||
1. Go to `https://git.levkin.ca/ilia/POTE/settings/secrets`
|
||||
2. Add each secret listed above
|
||||
|
||||
---
|
||||
|
||||
## 🟢 OPTIONAL - Customize for Your Needs
|
||||
|
||||
### 7. Email Recipients (Multiple locations)
|
||||
|
||||
**`.env` file:**
|
||||
```bash
|
||||
REPORT_RECIPIENTS=admin@localhost # 🟢 Change to your email(s), comma-separated
|
||||
```
|
||||
|
||||
**`scripts/automated_daily_run.sh`:**
|
||||
```bash
|
||||
REPORT_RECIPIENTS="${REPORT_RECIPIENTS:-admin@localhost}" # 🟢 Change default
|
||||
```
|
||||
|
||||
**`scripts/setup_cron.sh`:**
|
||||
- Will prompt you interactively for email address
|
||||
|
||||
**Action:** Update to your preferred email(s) for reports
|
||||
|
||||
### 8. Market Monitoring Tickers (`.env` and Ansible)
|
||||
|
||||
**`.env` file:**
|
||||
```bash
|
||||
MARKET_MONITOR_TICKERS=NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX
|
||||
# 🟢 Customize this list based on what Congress trades most
|
||||
```
|
||||
|
||||
**`ansible/group_vars/all.yml`:**
|
||||
```yaml
|
||||
market_tickers: "NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX"
|
||||
# 🟢 Should match .env
|
||||
```
|
||||
|
||||
**Action:** Research most-traded congressional stocks and update list
|
||||
|
||||
### 9. Alert Severity Threshold (`.env` and Ansible)
|
||||
|
||||
**`.env` file:**
|
||||
```bash
|
||||
ALERT_MIN_SEVERITY=5 # 🟢 1-10, lower = more sensitive
|
||||
```
|
||||
|
||||
**`ansible/group_vars/all.yml`:**
|
||||
```yaml
|
||||
alert_severity: 5 # 🟢 Should match .env
|
||||
```
|
||||
|
||||
**Action:** Adjust based on how many alerts you want (5 is moderate)
|
||||
|
||||
### 10. Cron Schedule Times (Ansible)
|
||||
|
||||
**`ansible/group_vars/development.yml`, `staging.yml`, `production.yml`:**
|
||||
```yaml
|
||||
pote_daily_report_time: "0 6" # 🟢 6:00 AM - change to your preference
|
||||
pote_weekly_report_time: "0 8" # 🟢 8:00 AM Sunday - change to your preference
|
||||
pote_health_check_time: "*/30 * * * *" # 🟢 Every 30 min - change to your preference
|
||||
```
|
||||
|
||||
**Action:** Adjust times based on your timezone and preferences
|
||||
|
||||
### 11. Log Level (`.env` and Ansible)
|
||||
|
||||
**`.env` file:**
|
||||
```bash
|
||||
LOG_LEVEL=INFO # 🟢 DEBUG, INFO, WARNING, ERROR
|
||||
```
|
||||
|
||||
**`ansible/group_vars/all.yml`:**
|
||||
```yaml
|
||||
log_level: "INFO" # 🟢 Should match .env
|
||||
```
|
||||
|
||||
**Action:** Use `DEBUG` for development, `INFO` for production
|
||||
|
||||
### 12. Application User (Ansible)
|
||||
|
||||
**`ansible/group_vars/all.yml`:**
|
||||
```yaml
|
||||
appuser_name: "poteapp" # 🟢 Change if you want a different username
|
||||
```
|
||||
|
||||
**`scripts/proxmox_setup.sh`:**
|
||||
```bash
|
||||
APP_USER="poteapp" # 🟢 Should match Ansible
|
||||
```
|
||||
|
||||
**Action:** Keep as `poteapp` unless you have a specific naming convention
|
||||
|
||||
### 13. Database Names (Ansible - per environment)
|
||||
|
||||
**`ansible/group_vars/development.yml`:**
|
||||
```yaml
|
||||
db_name: "potedb_dev" # 🟢 OK as-is
|
||||
```
|
||||
|
||||
**`ansible/group_vars/staging.yml`:**
|
||||
```yaml
|
||||
db_name: "potedb_qa" # 🟢 OK as-is
|
||||
```
|
||||
|
||||
**`ansible/group_vars/production.yml`:**
|
||||
```yaml
|
||||
db_name: "potedb" # 🟢 OK as-is
|
||||
```
|
||||
|
||||
**Action:** Keep defaults unless you have a specific naming convention
|
||||
|
||||
### 14. Backup Retention (Ansible)
|
||||
|
||||
**`ansible/roles/pote/defaults/main.yml`:**
|
||||
```yaml
|
||||
pote_backup_retention_days: 90 # 🟢 Adjust based on disk space
|
||||
```
|
||||
|
||||
**Action:** Increase for production (180+ days), decrease for dev (30 days)
|
||||
|
||||
### 15. API Keys (`.env` - if you get paid APIs)
|
||||
|
||||
**`.env` file:**
|
||||
```bash
|
||||
QUIVERQUANT_API_KEY= # 🟢 Optional paid service
|
||||
FMP_API_KEY= # 🟢 Optional paid service
|
||||
```
|
||||
|
||||
**Action:** Add keys if you subscribe to these services (not required)
|
||||
|
||||
---
|
||||
|
||||
## 📋 Summary: Quick Action List
|
||||
|
||||
### Immediate (Before First Deployment)
|
||||
1. ✅ Update `.env`: `SMTP_PASSWORD`, `REPORT_RECIPIENTS`, `DATABASE_URL` password
|
||||
2. ✅ Create `ansible/group_vars/all/vault.yml` with your SSH keys
|
||||
3. ✅ Encrypt vault: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
|
||||
4. ✅ Update `ansible/inventory.yml` with your server IPs
|
||||
5. ✅ Add Gitea secrets: `DB_PASSWORD`, `SMTP_PASSWORD`
|
||||
|
||||
### Before Production Use
|
||||
6. ✅ Verify Git repo URL in `ansible/group_vars/all.yml`
|
||||
7. ✅ Customize `MARKET_MONITOR_TICKERS` based on research
|
||||
8. ✅ Adjust cron times for your timezone
|
||||
9. ✅ Set appropriate log levels per environment
|
||||
|
||||
### Optional Enhancements
|
||||
10. ⭐ Add paid API keys if you subscribe
|
||||
11. ⭐ Adjust alert sensitivity based on testing
|
||||
12. ⭐ Customize backup retention per environment
|
||||
|
||||
---
|
||||
|
||||
## 🔍 How to Find These Files
|
||||
|
||||
```bash
|
||||
# Configuration files:
|
||||
.env # Main config (not in git)
|
||||
src/pote/config.py # Python config loader
|
||||
|
||||
# Ansible files:
|
||||
ansible/inventory.yml # Server IPs (copy from .example.yml)
|
||||
ansible/group_vars/all.yml # Common variables
|
||||
ansible/group_vars/all/vault.yml # Secrets (create from vault.example.yml)
|
||||
ansible/group_vars/development.yml # Dev environment
|
||||
ansible/group_vars/staging.yml # QA environment
|
||||
ansible/group_vars/production.yml # Prod environment
|
||||
ansible/roles/pote/defaults/main.yml # All default variables
|
||||
|
||||
# Scripts:
|
||||
scripts/automated_daily_run.sh # Daily automation
|
||||
scripts/automated_weekly_run.sh # Weekly automation
|
||||
scripts/setup_cron.sh # Cron setup
|
||||
scripts/proxmox_setup.sh # Initial server setup
|
||||
|
||||
# CI/CD:
|
||||
.github/workflows/ci.yml # CI pipeline
|
||||
.github/workflows/deploy.yml # Deployment pipeline
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚨 Security Reminders
|
||||
|
||||
1. **NEVER commit `.env` to git** - it's in `.gitignore`
|
||||
2. **NEVER commit unencrypted `vault.yml`** - always use `ansible-vault encrypt`
|
||||
3. **NEVER put real passwords in example files** - use placeholders
|
||||
4. **ALWAYS use strong passwords** - minimum 16 characters, mixed case, numbers, symbols
|
||||
5. **ALWAYS use Gitea secrets** for CI/CD - never hardcode in workflow files
|
||||
|
||||
---
|
||||
|
||||
## 📞 Need Help?
|
||||
|
||||
- **Ansible Vault:** `ansible-vault --help`
|
||||
- **Gitea Secrets:** Repository Settings → Secrets → Actions
|
||||
- **Environment Variables:** See `src/pote/config.py` for all available settings
|
||||
- **Testing Config:** Run `python scripts/health_check.py` after changes
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** December 2025
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
# POTE Deployment & Automation Guide
|
||||
|
||||
## Quick Answer to Your Questions
|
||||
## 🎯 Quick Answer to Your Questions
|
||||
|
||||
### After Deployment, What Happens?
|
||||
|
||||
**By default: NOTHING automatic happens.** You need to set up automation.
|
||||
|
||||
The deployed system is:
|
||||
- Running (database, code installed)
|
||||
- Accessible via SSH at your Proxmox IP
|
||||
- NOT fetching data automatically
|
||||
- NOT sending reports automatically
|
||||
- NOT monitoring markets automatically
|
||||
- ✅ Running (database, code installed)
|
||||
- ✅ Accessible via SSH at your Proxmox IP
|
||||
- ❌ NOT fetching data automatically
|
||||
- ❌ NOT sending reports automatically
|
||||
- ❌ NOT monitoring markets automatically
|
||||
|
||||
**You must either:**
|
||||
1. **Run scripts manually** when you want updates, OR
|
||||
@@ -19,7 +19,7 @@ The deployed system is:
|
||||
|
||||
---
|
||||
|
||||
## Option 1: Automated Email Reports (Recommended)
|
||||
## 🚀 Option 1: Automated Email Reports (Recommended)
|
||||
|
||||
### What You Get
|
||||
|
||||
@@ -53,13 +53,13 @@ Run the interactive setup:
|
||||
Follow prompts:
|
||||
1. Enter your email address
|
||||
2. Choose report time (default: 6 AM)
|
||||
3. Done!
|
||||
3. Done! ✅
|
||||
|
||||
**See full guide:** [`AUTOMATION_QUICKSTART.md`](AUTOMATION_QUICKSTART.md)
|
||||
|
||||
---
|
||||
|
||||
## Option 2: Access Reports via IP (No Email)
|
||||
## 📍 Option 2: Access Reports via IP (No Email)
|
||||
|
||||
If you don't want email, you can:
|
||||
|
||||
@@ -107,7 +107,7 @@ python scripts/health_check.py
|
||||
|
||||
---
|
||||
|
||||
## Option 3: Build a Web Interface (Future)
|
||||
## 🌐 Option 3: Build a Web Interface (Future)
|
||||
|
||||
Currently, POTE is **command-line only**. No web UI yet.
|
||||
|
||||
@@ -122,16 +122,16 @@ For now, use SSH or email reports.
|
||||
|
||||
---
|
||||
|
||||
## Do You Need CI/CD Pipelines?
|
||||
## 🔄 Do You Need CI/CD Pipelines?
|
||||
|
||||
### What the Pipeline Does
|
||||
|
||||
The included CI/CD pipeline (`.github/workflows/ci.yml`) runs on **every git push**:
|
||||
|
||||
1. Lint & test (93 tests)
|
||||
2. Security scanning
|
||||
3. Dependency scanning
|
||||
4. Docker build test
|
||||
1. ✅ Lint & test (93 tests)
|
||||
2. ✅ Security scanning
|
||||
3. ✅ Dependency scanning
|
||||
4. ✅ Docker build test
|
||||
|
||||
### Should You Use It?
|
||||
|
||||
@@ -174,18 +174,18 @@ docker build -t pote:test .
|
||||
|
||||
---
|
||||
|
||||
## Comparison of Options
|
||||
## 📊 Comparison of Options
|
||||
|
||||
| Method | Pros | Cons | Best For |
|
||||
|--------|------|------|----------|
|
||||
| **Automated Email** | Convenient<br> No SSH needed<br> Daily/weekly updates | Requires SMTP setup | Most users |
|
||||
| **SSH + Manual Scripts** | Full control<br> No email needed | Manual work<br> Must remember to run | Power users |
|
||||
| **Saved Reports (SSH access)** | Automated<br> No email | Must SSH to view | Users without email |
|
||||
| **Web Interface** | User-friendly | Not implemented yet | Future |
|
||||
| **Automated Email** | ✅ Convenient<br>✅ No SSH needed<br>✅ Daily/weekly updates | ❌ Requires SMTP setup | Most users |
|
||||
| **SSH + Manual Scripts** | ✅ Full control<br>✅ No email needed | ❌ Manual work<br>❌ Must remember to run | Power users |
|
||||
| **Saved Reports (SSH access)** | ✅ Automated<br>✅ No email | ❌ Must SSH to view | Users without email |
|
||||
| **Web Interface** | ✅ User-friendly | ❌ Not implemented yet | Future |
|
||||
|
||||
---
|
||||
|
||||
## Your Ansible Pipeline
|
||||
## 🛠️ Your Ansible Pipeline
|
||||
|
||||
Your existing Ansible CI/CD pipeline is **NOT directly usable** for POTE because:
|
||||
|
||||
@@ -197,11 +197,11 @@ Your existing Ansible CI/CD pipeline is **NOT directly usable** for POTE because
|
||||
|
||||
**Concepts from your pipeline that ARE used in POTE's CI/CD:**
|
||||
|
||||
- Security scanning (Trivy, Bandit instead of Gitleaks)
|
||||
- Dependency scanning (Trivy instead of npm audit)
|
||||
- SAST scanning (Bandit instead of Semgrep)
|
||||
- Container scanning (Docker build test)
|
||||
- Workflow summary generation
|
||||
- ✅ Security scanning (Trivy, Bandit instead of Gitleaks)
|
||||
- ✅ Dependency scanning (Trivy instead of npm audit)
|
||||
- ✅ SAST scanning (Bandit instead of Semgrep)
|
||||
- ✅ Container scanning (Docker build test)
|
||||
- ✅ Workflow summary generation
|
||||
|
||||
**The POTE pipeline (`.github/workflows/ci.yml`) already includes all of these!**
|
||||
|
||||
@@ -217,7 +217,7 @@ Your Gitea server can run the POTE pipeline using Gitea Actions:
|
||||
|
||||
---
|
||||
|
||||
## Email Setup Examples
|
||||
## 📧 Email Setup Examples
|
||||
|
||||
### Gmail (Most Common)
|
||||
|
||||
@@ -258,7 +258,7 @@ REPORT_RECIPIENTS=admin@yourdomain.com
|
||||
|
||||
---
|
||||
|
||||
## Recommended Setup for Most Users
|
||||
## ✅ Recommended Setup for Most Users
|
||||
|
||||
1. **Deploy to Proxmox** (5 min)
|
||||
```bash
|
||||
@@ -283,7 +283,7 @@ REPORT_RECIPIENTS=admin@yourdomain.com
|
||||
|
||||
---
|
||||
|
||||
## Monitoring & Health Checks
|
||||
## 🔍 Monitoring & Health Checks
|
||||
|
||||
### Add System Health Monitoring
|
||||
|
||||
@@ -310,17 +310,17 @@ python scripts/health_check.py
|
||||
|
||||
---
|
||||
|
||||
## Full Documentation
|
||||
## 📚 Full Documentation
|
||||
|
||||
- **Automation Setup**: [`AUTOMATION_QUICKSTART.md`](AUTOMATION_QUICKSTART.md)
|
||||
- **Deployment**: [`PROXMOX_QUICKSTART.md`](PROXMOX_QUICKSTART.md)
|
||||
- **Usage**: [`QUICKSTART.md`](QUICKSTART.md)
|
||||
- **Automation Setup**: [`AUTOMATION_QUICKSTART.md`](AUTOMATION_QUICKSTART.md) ⭐
|
||||
- **Deployment**: [`PROXMOX_QUICKSTART.md`](PROXMOX_QUICKSTART.md) ⭐
|
||||
- **Usage**: [`QUICKSTART.md`](QUICKSTART.md) ⭐
|
||||
- **Detailed Automation Guide**: [`docs/12_automation_and_reporting.md`](docs/12_automation_and_reporting.md)
|
||||
- **Monitoring System**: [`MONITORING_SYSTEM_COMPLETE.md`](MONITORING_SYSTEM_COMPLETE.md)
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
## 🎉 Summary
|
||||
|
||||
**After deployment:**
|
||||
- Reports are NOT sent automatically by default
|
||||
@@ -334,5 +334,5 @@ python scripts/health_check.py
|
||||
1. Deploy to Proxmox
|
||||
2. Run `./scripts/setup_cron.sh`
|
||||
3. Receive daily/weekly email reports
|
||||
4. Done!
|
||||
4. Done! 🚀
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
# Email Setup for levkine.ca (Mailcow)
|
||||
# Email Setup for levkin.ca
|
||||
|
||||
Homelab POTE sends via Mailcow **`mail.levkine.ca`** using the shared **`alerts@levkine.ca`** mailbox (same as Kuma/Beszel). See ansible `docs/guides/smtp-inventory.md`.
|
||||
Your POTE system is configured to use `test@levkin.ca` for sending reports.
|
||||
|
||||
## Configuration Done
|
||||
## ✅ Configuration Done
|
||||
|
||||
The `.env` file has been created with these settings:
|
||||
|
||||
```env
|
||||
SMTP_HOST=10.0.10.132
|
||||
SMTP_HOST=mail.levkin.ca
|
||||
SMTP_PORT=587
|
||||
SMTP_USER=alerts@levkine.ca
|
||||
SMTP_USER=test@levkin.ca
|
||||
SMTP_PASSWORD=YOUR_MAILBOX_PASSWORD_HERE
|
||||
FROM_EMAIL=alerts@levkine.ca
|
||||
REPORT_RECIPIENTS=idobkin@gmail.com
|
||||
FROM_EMAIL=test@levkin.ca
|
||||
REPORT_RECIPIENTS=test@levkin.ca
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
## 🔑 Next Steps
|
||||
|
||||
### 1. Add Your Password
|
||||
|
||||
@@ -38,7 +38,7 @@ python scripts/send_daily_report.py --to test@levkin.ca --test-smtp
|
||||
If successful, you'll see:
|
||||
```
|
||||
SMTP connection test successful!
|
||||
Daily report sent successfully!
|
||||
✓ Daily report sent successfully!
|
||||
```
|
||||
|
||||
And you should receive a test email at `test@levkin.ca`!
|
||||
@@ -57,7 +57,7 @@ This will:
|
||||
- Schedule daily reports (default: 6 AM)
|
||||
- Schedule weekly reports (Sundays at 8 AM)
|
||||
|
||||
## Email Server Details (For Reference)
|
||||
## 📧 Email Server Details (For Reference)
|
||||
|
||||
Based on your Thunderbird setup:
|
||||
|
||||
@@ -76,10 +76,10 @@ Based on your Thunderbird setup:
|
||||
|
||||
POTE only uses **SMTP (outgoing)** to send reports.
|
||||
|
||||
## Security Notes
|
||||
## 🔒 Security Notes
|
||||
|
||||
1. **Never commit `.env` to git!**
|
||||
- Already in `.gitignore`
|
||||
- Already in `.gitignore` ✅
|
||||
- Contains sensitive password
|
||||
|
||||
2. **Password Security:**
|
||||
@@ -92,7 +92,7 @@ POTE only uses **SMTP (outgoing)** to send reports.
|
||||
chmod 600 .env # Only owner can read/write
|
||||
```
|
||||
|
||||
## Change Recipients
|
||||
## 🎯 Change Recipients
|
||||
|
||||
To send reports to different email addresses (not just test@levkin.ca):
|
||||
|
||||
@@ -109,14 +109,14 @@ python scripts/send_daily_report.py --to someone-else@example.com
|
||||
# The FROM address will still be test@levkin.ca
|
||||
```
|
||||
|
||||
## Testing Checklist
|
||||
## ✅ Testing Checklist
|
||||
|
||||
- [ ] Updated `.env` with your actual password
|
||||
- [ ] Run `python scripts/send_daily_report.py --to test@levkin.ca --test-smtp`
|
||||
- [ ] Checked inbox at test@levkin.ca (check spam folder!)
|
||||
- [ ] If successful, run `./scripts/setup_cron.sh` to automate
|
||||
|
||||
## Troubleshooting
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### Error: "SMTP connection failed"
|
||||
|
||||
@@ -139,5 +139,5 @@ Check:
|
||||
|
||||
---
|
||||
|
||||
**You're all set! POTE will send reports from test@levkin.ca **
|
||||
**You're all set! POTE will send reports from test@levkin.ca 📧**
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
## TL;DR: You can test everything for $0
|
||||
|
||||
### Already Working (PR1 )
|
||||
### Already Working (PR1 ✅)
|
||||
- **Price data**: `yfinance` (free, unlimited)
|
||||
- **Unit tests**: Mocked data in `tests/` (15 passing tests)
|
||||
- **Coverage**: 87% without any paid APIs
|
||||
|
||||
### For PR2 (Congressional Trades) - FREE Options
|
||||
|
||||
#### Best Option: House Stock Watcher
|
||||
#### Best Option: House Stock Watcher 🌟
|
||||
```bash
|
||||
# No API key needed, just scrape their public JSON
|
||||
curl https://housestockwatcher.com/api/all_transactions
|
||||
@@ -44,10 +44,10 @@ python scripts/fetch_house_watcher_sample.py # We'll build this in PR2
|
||||
|
||||
### What You DON'T Need to Pay For
|
||||
|
||||
QuiverQuant Pro ($30/mo) - free tier is enough for dev/testing
|
||||
Financial Modeling Prep paid tier - free tier works
|
||||
Any paid database hosting - SQLite works great locally
|
||||
Any cloud services - runs 100% locally
|
||||
❌ QuiverQuant Pro ($30/mo) - free tier is enough for dev/testing
|
||||
❌ Financial Modeling Prep paid tier - free tier works
|
||||
❌ Any paid database hosting - SQLite works great locally
|
||||
❌ Any cloud services - runs 100% locally
|
||||
|
||||
### When You MIGHT Want Paid (Way Later)
|
||||
|
||||
@@ -56,7 +56,7 @@ python scripts/fetch_house_watcher_sample.py # We'll build this in PR2
|
||||
- Multiple concurrent users on a dashboard
|
||||
- Commercial use (check each API's terms)
|
||||
|
||||
**For personal research? Stay free forever. **
|
||||
**For personal research? Stay free forever. 🎉**
|
||||
|
||||
---
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
# Gitea Secrets Guide for POTE
|
||||
# 🔐 Gitea Secrets Guide for POTE
|
||||
|
||||
## YES! You Can Store Passwords in Gitea
|
||||
## ✅ YES! You Can Store Passwords in Gitea
|
||||
|
||||
Gitea has a **Secrets** feature (like GitHub Actions secrets) that lets you store passwords securely and use them in:
|
||||
1. **CI/CD pipelines** (Gitea Actions workflows)
|
||||
2. **Deployment workflows**
|
||||
1. **CI/CD pipelines** (Gitea Actions workflows) ✅
|
||||
2. **Deployment workflows** ✅
|
||||
|
||||
**BUT NOT:**
|
||||
- Directly in your running application on Proxmox
|
||||
- Accessed by scripts outside of workflows
|
||||
- ❌ Directly in your running application on Proxmox
|
||||
- ❌ Accessed by scripts outside of workflows
|
||||
|
||||
---
|
||||
|
||||
## What Gitea Secrets Are Good For
|
||||
## 🎯 What Gitea Secrets Are Good For
|
||||
|
||||
### Perfect Use Cases
|
||||
### ✅ Perfect Use Cases
|
||||
|
||||
1. **CI/CD Testing** - Run tests with real credentials
|
||||
2. **Automated Deployment** - Deploy to Proxmox with SSH keys
|
||||
@@ -22,7 +22,7 @@ Gitea has a **Secrets** feature (like GitHub Actions secrets) that lets you stor
|
||||
4. **Docker Registry** - Push images with credentials
|
||||
5. **API Keys** - Access external services during builds
|
||||
|
||||
### NOT Good For
|
||||
### ❌ NOT Good For
|
||||
|
||||
1. **Runtime secrets** - Your deployed app on Proxmox can't access them
|
||||
2. **Local development** - Can't use secrets on your laptop
|
||||
@@ -30,7 +30,7 @@ Gitea has a **Secrets** feature (like GitHub Actions secrets) that lets you stor
|
||||
|
||||
---
|
||||
|
||||
## How to Set Up Gitea Secrets
|
||||
## 🔧 How to Set Up Gitea Secrets
|
||||
|
||||
### Step 1: Add Secrets to Gitea
|
||||
|
||||
@@ -57,7 +57,7 @@ Secrets are accessed with `${{ secrets.SECRET_NAME }}` syntax.
|
||||
|
||||
---
|
||||
|
||||
## Example: CI Pipeline with Secrets
|
||||
## 📝 Example: CI Pipeline with Secrets
|
||||
|
||||
**File:** `.github/workflows/ci.yml`
|
||||
|
||||
@@ -92,11 +92,11 @@ jobs:
|
||||
--smtp-password "${{ secrets.SMTP_PASSWORD }}"
|
||||
```
|
||||
|
||||
** I've already updated your CI pipeline to use secrets!**
|
||||
**✅ I've already updated your CI pipeline to use secrets!**
|
||||
|
||||
---
|
||||
|
||||
## Example: Automated Deployment Workflow
|
||||
## 🚀 Example: Automated Deployment Workflow
|
||||
|
||||
Create `.github/workflows/deploy.yml`:
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
---
|
||||
|
||||
## How Secrets Flow to Your Server
|
||||
## 🔄 How Secrets Flow to Your Server
|
||||
|
||||
### Option 1: Deploy Workflow Updates `.env` (Recommended)
|
||||
|
||||
@@ -201,11 +201,11 @@ jobs:
|
||||
|
||||
---
|
||||
|
||||
## Recommended Setup for Your POTE Project
|
||||
## 🎯 Recommended Setup for Your POTE Project
|
||||
|
||||
### For CI/CD (Testing):
|
||||
|
||||
**Use Gitea Secrets**
|
||||
**Use Gitea Secrets** ✅
|
||||
|
||||
```yaml
|
||||
# .github/workflows/ci.yml (already updated!)
|
||||
@@ -216,7 +216,7 @@ env:
|
||||
|
||||
### For Deployed Server (Proxmox):
|
||||
|
||||
**Keep using `.env` file**
|
||||
**Keep using `.env` file** ✅
|
||||
|
||||
Why?
|
||||
- Simpler for manual SSH access
|
||||
@@ -227,7 +227,7 @@ Why?
|
||||
|
||||
---
|
||||
|
||||
## Complete Workflow: Gitea → Proxmox
|
||||
## 🚀 Complete Workflow: Gitea → Proxmox
|
||||
|
||||
### 1. Store Secrets in Gitea
|
||||
|
||||
@@ -264,27 +264,27 @@ git push origin main
|
||||
|
||||
---
|
||||
|
||||
## Important Limitations
|
||||
## ⚠️ Important Limitations
|
||||
|
||||
### Gitea Secrets CAN'T:
|
||||
|
||||
Be accessed outside of workflows
|
||||
Be used in local `python script.py` runs
|
||||
Be read by cron jobs on Proxmox (directly)
|
||||
Replace `.env` for runtime application config
|
||||
❌ Be accessed outside of workflows
|
||||
❌ Be used in local `python script.py` runs
|
||||
❌ Be read by cron jobs on Proxmox (directly)
|
||||
❌ Replace `.env` for runtime application config
|
||||
|
||||
### Gitea Secrets CAN:
|
||||
|
||||
Secure your CI/CD pipeline
|
||||
Deploy safely without exposing passwords in git
|
||||
Update `.env` on server during deployment
|
||||
Run automated tests with real credentials
|
||||
✅ Secure your CI/CD pipeline
|
||||
✅ Deploy safely without exposing passwords in git
|
||||
✅ Update `.env` on server during deployment
|
||||
✅ Run automated tests with real credentials
|
||||
|
||||
---
|
||||
|
||||
## Security Best Practices
|
||||
## 🔒 Security Best Practices
|
||||
|
||||
### DO:
|
||||
### ✅ DO:
|
||||
|
||||
1. **Store ALL sensitive data as Gitea secrets**
|
||||
- SMTP passwords
|
||||
@@ -300,10 +300,10 @@ git push origin main
|
||||
|
||||
3. **Never echo secrets**
|
||||
```yaml
|
||||
# BAD - exposes in logs
|
||||
# ❌ BAD - exposes in logs
|
||||
- run: echo "${{ secrets.PASSWORD }}"
|
||||
|
||||
# GOOD - masked automatically
|
||||
# ✅ GOOD - masked automatically
|
||||
- run: use_password "${{ secrets.PASSWORD }}"
|
||||
```
|
||||
|
||||
@@ -311,7 +311,7 @@ git push origin main
|
||||
- Update in Gitea UI
|
||||
- Re-run deployment workflow
|
||||
|
||||
### DON'T:
|
||||
### ❌ DON'T:
|
||||
|
||||
1. **Commit secrets to git** (even private repos)
|
||||
2. **Share secrets via Slack/email**
|
||||
@@ -320,18 +320,18 @@ git push origin main
|
||||
|
||||
---
|
||||
|
||||
## Comparison: Where to Store Secrets
|
||||
## 📊 Comparison: Where to Store Secrets
|
||||
|
||||
| Storage | CI/CD | Deployed App | Easy Updates | Security |
|
||||
|---------|-------|--------------|--------------|----------|
|
||||
| **Gitea Secrets** | Perfect | No | Via workflow | |
|
||||
| **`.env` file** | No | Perfect | `nano .env` | |
|
||||
| **Environment Vars** | Yes | Yes | Harder | |
|
||||
| **Both (Recommended)** | Yes | Yes | Automated | |
|
||||
| **Gitea Secrets** | ✅ Perfect | ❌ No | ✅ Via workflow | ⭐⭐⭐⭐⭐ |
|
||||
| **`.env` file** | ❌ No | ✅ Perfect | ✅ `nano .env` | ⭐⭐⭐ |
|
||||
| **Environment Vars** | ✅ Yes | ✅ Yes | ❌ Harder | ⭐⭐⭐⭐ |
|
||||
| **Both (Recommended)** | ✅ Yes | ✅ Yes | ✅ Automated | ⭐⭐⭐⭐⭐ |
|
||||
|
||||
---
|
||||
|
||||
## My Recommendation for You
|
||||
## 🎯 My Recommendation for You
|
||||
|
||||
### Use BOTH:
|
||||
|
||||
@@ -345,21 +345,21 @@ git push origin main
|
||||
2. Commit code changes
|
||||
3. Push to Gitea
|
||||
4. Workflow runs:
|
||||
- Tests with Gitea secrets
|
||||
- Deploys to Proxmox
|
||||
- Updates .env with secrets
|
||||
5. Proxmox app reads from .env
|
||||
- Tests with Gitea secrets ✅
|
||||
- Deploys to Proxmox ✅
|
||||
- Updates .env with secrets ✅
|
||||
5. Proxmox app reads from .env ✅
|
||||
```
|
||||
|
||||
**This gives you:**
|
||||
- Secure CI/CD
|
||||
- Easy manual SSH access
|
||||
- Automated deployments
|
||||
- No passwords in git
|
||||
- ✅ Secure CI/CD
|
||||
- ✅ Easy manual SSH access
|
||||
- ✅ Automated deployments
|
||||
- ✅ No passwords in git
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
## 🚀 Next Steps
|
||||
|
||||
### 1. Add Secrets to Gitea (5 minutes)
|
||||
|
||||
@@ -387,7 +387,7 @@ I can create `.github/workflows/deploy.yml` if you want automated deployments!
|
||||
|
||||
---
|
||||
|
||||
## Quick Commands
|
||||
## 💡 Quick Commands
|
||||
|
||||
### Add SSH Key to Gitea (for deployment):
|
||||
|
||||
@@ -409,12 +409,12 @@ git commit --allow-empty -m "Test secrets"
|
||||
git push
|
||||
|
||||
# Check Gitea Actions tab
|
||||
# Look for green checkmarks
|
||||
# Look for green checkmarks ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## See Also
|
||||
## 📚 See Also
|
||||
|
||||
- **[docs/13_secrets_management.md](docs/13_secrets_management.md)** - All secrets options
|
||||
- **[.github/workflows/ci.yml](.github/workflows/ci.yml)** - Updated with secrets support
|
||||
@@ -422,16 +422,16 @@ git push
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
## ✅ Summary
|
||||
|
||||
**YES, use Gitea secrets!** They're perfect for:
|
||||
- CI/CD pipelines
|
||||
- Automated deployments
|
||||
- Keeping passwords out of git
|
||||
- ✅ CI/CD pipelines
|
||||
- ✅ Automated deployments
|
||||
- ✅ Keeping passwords out of git
|
||||
|
||||
**But ALSO keep `.env` on Proxmox** for:
|
||||
- Runtime application config
|
||||
- Manual SSH access
|
||||
- Cron jobs
|
||||
- ✅ Runtime application config
|
||||
- ✅ Manual SSH access
|
||||
- ✅ Cron jobs
|
||||
|
||||
**Best of both worlds:** Gitea secrets deploy and update the `.env` file automatically!
|
||||
**Best of both worlds:** Gitea secrets deploy and update the `.env` file automatically! 🚀
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Ilia Dobkin
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,6 +1,6 @@
|
||||
# Local Testing Guide for POTE
|
||||
|
||||
## Testing Locally Before Deployment
|
||||
## ✅ Testing Locally Before Deployment
|
||||
|
||||
### Quick Test - Run Full Suite
|
||||
|
||||
@@ -10,18 +10,18 @@ source venv/bin/activate
|
||||
pytest -v
|
||||
```
|
||||
|
||||
**Expected Result:** All 55 tests should pass
|
||||
**Expected Result:** All 55 tests should pass ✅
|
||||
|
||||
---
|
||||
|
||||
## Current Data Status
|
||||
## 📊 Current Data Status
|
||||
|
||||
### Live Data Status: **NOT LIVE YET**
|
||||
### Live Data Status: ❌ **NOT LIVE YET**
|
||||
|
||||
**Why?**
|
||||
- **House Stock Watcher API is DOWN** (domain issues, unreachable)
|
||||
- **yfinance works** (for price data)
|
||||
- **Sample data available** (5 trades from fixtures)
|
||||
- 🔴 **House Stock Watcher API is DOWN** (domain issues, unreachable)
|
||||
- 🟢 **yfinance works** (for price data)
|
||||
- 🟡 **Sample data available** (5 trades from fixtures)
|
||||
|
||||
### What Data Do You Have?
|
||||
|
||||
@@ -40,7 +40,7 @@ This will show:
|
||||
|
||||
---
|
||||
|
||||
## Testing Analytics Locally
|
||||
## 🧪 Testing Analytics Locally
|
||||
|
||||
### 1. Unit Tests (Fast, No External Dependencies)
|
||||
|
||||
@@ -53,10 +53,10 @@ pytest tests/test_analytics_integration.py -v
|
||||
```
|
||||
|
||||
These tests:
|
||||
- Create synthetic price data
|
||||
- Simulate trades with known returns
|
||||
- Verify calculations are correct
|
||||
- Test edge cases (missing data, sell trades, etc.)
|
||||
- ✅ Create synthetic price data
|
||||
- ✅ Simulate trades with known returns
|
||||
- ✅ Verify calculations are correct
|
||||
- ✅ Test edge cases (missing data, sell trades, etc.)
|
||||
|
||||
### 2. Manual Test with Local Database
|
||||
|
||||
@@ -92,27 +92,27 @@ calc = ReturnCalculator(session)
|
||||
|
||||
---
|
||||
|
||||
## What Gets Tested?
|
||||
## 📦 What Gets Tested?
|
||||
|
||||
### Core Functionality (All Working )
|
||||
### Core Functionality (All Working ✅)
|
||||
1. **Database Models** - Officials, Securities, Trades, Prices
|
||||
2. **Data Ingestion** - Trade loading, security enrichment
|
||||
3. **Analytics Engine** - Returns, benchmarks, metrics
|
||||
4. **Edge Cases** - Missing data, sell trades, disclosure lags
|
||||
|
||||
### Integration Tests Cover:
|
||||
- Return calculations over multiple time windows (30/60/90/180 days)
|
||||
- Benchmark comparisons (stock vs SPY/QQQ)
|
||||
- Abnormal return (alpha) calculations
|
||||
- Official performance summaries
|
||||
- Sector analysis
|
||||
- Disclosure timing analysis
|
||||
- Top performer rankings
|
||||
- System-wide statistics
|
||||
- ✅ Return calculations over multiple time windows (30/60/90/180 days)
|
||||
- ✅ Benchmark comparisons (stock vs SPY/QQQ)
|
||||
- ✅ Abnormal return (alpha) calculations
|
||||
- ✅ Official performance summaries
|
||||
- ✅ Sector analysis
|
||||
- ✅ Disclosure timing analysis
|
||||
- ✅ Top performer rankings
|
||||
- ✅ System-wide statistics
|
||||
|
||||
---
|
||||
|
||||
## Getting Live Data
|
||||
## 🔄 Getting Live Data
|
||||
|
||||
### Option 1: Wait for House Stock Watcher API
|
||||
The API is currently down. Once it's back up:
|
||||
@@ -164,7 +164,7 @@ export QUIVER_API_KEY="your_key_here"
|
||||
|
||||
---
|
||||
|
||||
## After Adding Data, Fetch Prices
|
||||
## 📈 After Adding Data, Fetch Prices
|
||||
|
||||
```bash
|
||||
# This will fetch prices for all securities in your database
|
||||
@@ -176,12 +176,12 @@ python scripts/enrich_securities.py
|
||||
|
||||
---
|
||||
|
||||
## Complete Local Test Workflow
|
||||
## 🎯 Complete Local Test Workflow
|
||||
|
||||
```bash
|
||||
# 1. Run all tests
|
||||
pytest -v
|
||||
# All 55 tests should pass
|
||||
# ✅ All 55 tests should pass
|
||||
|
||||
# 2. Check local database
|
||||
python -c "
|
||||
@@ -210,7 +210,7 @@ python scripts/calculate_all_returns.py --window 90
|
||||
|
||||
---
|
||||
|
||||
## Deploy to Proxmox
|
||||
## 🚀 Deploy to Proxmox
|
||||
|
||||
Once local tests pass:
|
||||
|
||||
@@ -240,7 +240,7 @@ alembic upgrade head
|
||||
|
||||
---
|
||||
|
||||
## Common Issues
|
||||
## 🐛 Common Issues
|
||||
|
||||
### "No price data found"
|
||||
**Fix:** Run `python scripts/fetch_sample_prices.py`
|
||||
@@ -260,7 +260,7 @@ sudo -u postgres psql -c "\l"
|
||||
|
||||
---
|
||||
|
||||
## Test Coverage
|
||||
## 📊 Test Coverage
|
||||
|
||||
Run tests with coverage report:
|
||||
|
||||
@@ -277,19 +277,19 @@ firefox htmlcov/index.html # View coverage report
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
## ✨ Summary
|
||||
|
||||
**Before Deploying:**
|
||||
1. Run `pytest -v` - all tests pass
|
||||
2. Run `make lint` - no errors
|
||||
3. Test locally with sample data
|
||||
4. Verify analytics work with synthetic prices
|
||||
1. ✅ Run `pytest -v` - all tests pass
|
||||
2. ✅ Run `make lint` - no errors
|
||||
3. ✅ Test locally with sample data
|
||||
4. ✅ Verify analytics work with synthetic prices
|
||||
|
||||
**Getting Live Data:**
|
||||
- House Stock Watcher API is down (external issue)
|
||||
- Manual CSV import works NOW
|
||||
- yfinance for prices works NOW
|
||||
- QuiverQuant available (requires free API key)
|
||||
- 🔴 House Stock Watcher API is down (external issue)
|
||||
- 🟢 Manual CSV import works NOW
|
||||
- 🟢 yfinance for prices works NOW
|
||||
- 🟡 QuiverQuant available (requires free API key)
|
||||
|
||||
**You can deploy and use the system NOW with:**
|
||||
- Manual data entry
|
||||
@@ -1,8 +1,8 @@
|
||||
# POTE Monitoring System - ALL PHASES COMPLETE!
|
||||
# 🎉 POTE Monitoring System - ALL PHASES COMPLETE!
|
||||
|
||||
## **What Was Built (3 Phases)**
|
||||
## ✅ **What Was Built (3 Phases)**
|
||||
|
||||
### **Phase 1: Real-Time Market Monitoring**
|
||||
### **Phase 1: Real-Time Market Monitoring** ✅
|
||||
**Detects unusual market activity in congressional tickers**
|
||||
|
||||
**Features:**
|
||||
@@ -20,11 +20,11 @@
|
||||
- `MarketAlert` model - Database storage
|
||||
- `monitor_market.py` - CLI tool
|
||||
|
||||
**Tests:** 14 passing
|
||||
**Tests:** 14 passing ✅
|
||||
|
||||
---
|
||||
|
||||
### **Phase 2: Disclosure Timing Correlation**
|
||||
### **Phase 2: Disclosure Timing Correlation** ✅
|
||||
**Matches trades to prior market alerts when disclosures appear**
|
||||
|
||||
**Features:**
|
||||
@@ -50,11 +50,11 @@
|
||||
- `DisclosureCorrelator` - Correlation engine
|
||||
- `analyze_disclosure_timing.py` - CLI tool
|
||||
|
||||
**Tests:** 13 passing
|
||||
**Tests:** 13 passing ✅
|
||||
|
||||
---
|
||||
|
||||
### **Phase 3: Pattern Detection & Rankings**
|
||||
### **Phase 3: Pattern Detection & Rankings** ✅
|
||||
**Cross-official analysis and comparative rankings**
|
||||
|
||||
**Features:**
|
||||
@@ -71,19 +71,19 @@
|
||||
- `PatternDetector` - Pattern analysis engine
|
||||
- `generate_pattern_report.py` - CLI tool
|
||||
|
||||
**Tests:** 11 passing
|
||||
**Tests:** 11 passing ✅
|
||||
|
||||
---
|
||||
|
||||
## **Complete System Architecture**
|
||||
## 📊 **Complete System Architecture**
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ PHASE 1: Real-Time Monitoring │
|
||||
│ ──────────────────────────────────── │
|
||||
│ Monitor congressional tickers │
|
||||
│ Detect unusual activity │
|
||||
│ Log alerts to database │
|
||||
│ 🔔 Monitor congressional tickers │
|
||||
│ 📊 Detect unusual activity │
|
||||
│ 💾 Log alerts to database │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
[30-45 days pass]
|
||||
@@ -91,25 +91,25 @@
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ PHASE 2: Disclosure Correlation │
|
||||
│ ─────────────────────────────── │
|
||||
│ New congressional trades filed │
|
||||
│ Match to prior alerts │
|
||||
│ Calculate timing scores │
|
||||
│ Flag suspicious trades │
|
||||
│ 📋 New congressional trades filed │
|
||||
│ 🔗 Match to prior alerts │
|
||||
│ 📈 Calculate timing scores │
|
||||
│ 🚩 Flag suspicious trades │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ PHASE 3: Pattern Detection │
|
||||
│ ────────────────────────── │
|
||||
│ Rank officials by timing │
|
||||
│ Identify repeat offenders │
|
||||
│ Compare parties, sectors, tickers │
|
||||
│ Generate comprehensive reports │
|
||||
│ 📊 Rank officials by timing │
|
||||
│ 🔥 Identify repeat offenders │
|
||||
│ 📈 Compare parties, sectors, tickers │
|
||||
│ 📋 Generate comprehensive reports │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## **Usage Guide**
|
||||
## 🚀 **Usage Guide**
|
||||
|
||||
### **1. Set Up Monitoring (Run Daily)**
|
||||
|
||||
@@ -169,7 +169,7 @@ python scripts/generate_pattern_report.py --days 365 --format json --output patt
|
||||
|
||||
---
|
||||
|
||||
## **Example Reports**
|
||||
## 📋 **Example Reports**
|
||||
|
||||
### **Timing Analysis Report**
|
||||
|
||||
@@ -179,7 +179,7 @@ python scripts/generate_pattern_report.py --days 365 --format json --output patt
|
||||
3 Trades with Timing Advantages Detected
|
||||
================================================================================
|
||||
|
||||
#1 - HIGHLY SUSPICIOUS (Timing Score: 85/100)
|
||||
🚨 #1 - HIGHLY SUSPICIOUS (Timing Score: 85/100)
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Official: Nancy Pelosi
|
||||
Ticker: NVDA
|
||||
@@ -187,16 +187,16 @@ Side: BUY
|
||||
Trade Date: 2024-01-15
|
||||
Value: $15,001-$50,000
|
||||
|
||||
Timing Analysis:
|
||||
📊 Timing Analysis:
|
||||
Prior Alerts: 3
|
||||
Recent Alerts (7d): 2
|
||||
High Severity: 2
|
||||
Avg Severity: 7.5/10
|
||||
|
||||
Assessment: Trade occurred after 3 alerts, including 2 high-severity.
|
||||
💡 Assessment: Trade occurred after 3 alerts, including 2 high-severity.
|
||||
High likelihood of timing advantage.
|
||||
|
||||
Prior Market Alerts:
|
||||
🔔 Prior Market Alerts:
|
||||
Timestamp Type Severity Timing
|
||||
2024-01-12 10:30:00 Unusual Volume 8/10 3 days before
|
||||
2024-01-13 14:15:00 Price Spike 7/10 2 days before
|
||||
@@ -211,27 +211,27 @@ Timestamp Type Severity Timing
|
||||
Period: 365 days
|
||||
================================================================================
|
||||
|
||||
SUMMARY
|
||||
📊 SUMMARY
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Officials Analyzed: 45
|
||||
Repeat Offenders: 8
|
||||
Average Timing Score: 42.3/100
|
||||
|
||||
TOP 10 MOST SUSPICIOUS OFFICIALS (By Timing Score)
|
||||
🚨 TOP 10 MOST SUSPICIOUS OFFICIALS (By Timing Score)
|
||||
================================================================================
|
||||
|
||||
Rank Official Party-State Chamber Trades Suspicious Rate Avg Score
|
||||
──── ─────────────────────── ─────────── ─────── ────── ────────── ────── ─────────
|
||||
1 Tommy Tuberville R-AL Senate 47 35/47 74.5% 72.5/100
|
||||
2 Nancy Pelosi D-CA House 38 28/38 73.7% 71.2/100
|
||||
3 Dan Crenshaw R-TX House 25 15/25 60.0% 65.8/100
|
||||
4 Marjorie Taylor Greene R-GA House 19 11/19 57.9% 63.2/100
|
||||
5 Josh Gottheimer D-NJ House 31 14/31 45.2% 58.7/100
|
||||
🚨 1 Tommy Tuberville R-AL Senate 47 35/47 74.5% 72.5/100
|
||||
🚨 2 Nancy Pelosi D-CA House 38 28/38 73.7% 71.2/100
|
||||
🔴 3 Dan Crenshaw R-TX House 25 15/25 60.0% 65.8/100
|
||||
🔴 4 Marjorie Taylor Greene R-GA House 19 11/19 57.9% 63.2/100
|
||||
🟡 5 Josh Gottheimer D-NJ House 31 14/31 45.2% 58.7/100
|
||||
|
||||
REPEAT OFFENDERS (50%+ Suspicious Trades)
|
||||
🔥 REPEAT OFFENDERS (50%+ Suspicious Trades)
|
||||
================================================================================
|
||||
|
||||
Tommy Tuberville (R-AL, Senate)
|
||||
🚨 Tommy Tuberville (R-AL, Senate)
|
||||
Trades: 47 | Suspicious: 35 (74.5%)
|
||||
Avg Timing Score: 72.5/100
|
||||
Pattern: HIGHLY SUSPICIOUS - Majority of trades show timing advantage
|
||||
@@ -239,9 +239,9 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Test Coverage**
|
||||
## 📈 **Test Coverage**
|
||||
|
||||
**Total: 93 tests, all passing **
|
||||
**Total: 93 tests, all passing ✅**
|
||||
|
||||
- **Phase 1 (Monitoring):** 14 tests
|
||||
- **Phase 2 (Correlation):** 13 tests
|
||||
@@ -252,7 +252,7 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Key Insights the System Provides**
|
||||
## 🎯 **Key Insights the System Provides**
|
||||
|
||||
### **1. Individual Official Analysis**
|
||||
- Which officials consistently trade before unusual activity?
|
||||
@@ -282,7 +282,7 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Automated Workflow**
|
||||
## 🔧 **Automated Workflow**
|
||||
|
||||
### **Daily Routine (Recommended)**
|
||||
|
||||
@@ -299,7 +299,7 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Database Schema**
|
||||
## 📊 **Database Schema**
|
||||
|
||||
**New Table: `market_alerts`**
|
||||
```sql
|
||||
@@ -315,7 +315,7 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Interpretation Guide**
|
||||
## 🎓 **Interpretation Guide**
|
||||
|
||||
### **Timing Scores**
|
||||
- **80-100:** Highly suspicious - Multiple high-severity alerts before trade
|
||||
@@ -337,15 +337,15 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Important Disclaimers**
|
||||
## ⚠️ **Important Disclaimers**
|
||||
|
||||
### **Legal & Ethical**
|
||||
1. All data is public and legally obtained
|
||||
2. Analysis is retrospective (30-45 day lag)
|
||||
3. For research and transparency only
|
||||
4. NOT investment advice
|
||||
5. NOT proof of illegal activity (requires investigation)
|
||||
6. Statistical patterns ≠ legal evidence
|
||||
1. ✅ All data is public and legally obtained
|
||||
2. ✅ Analysis is retrospective (30-45 day lag)
|
||||
3. ✅ For research and transparency only
|
||||
4. ❌ NOT investment advice
|
||||
5. ❌ NOT proof of illegal activity (requires investigation)
|
||||
6. ❌ Statistical patterns ≠ legal evidence
|
||||
|
||||
### **Technical Limitations**
|
||||
1. Cannot identify WHO is trading in real-time
|
||||
@@ -356,7 +356,7 @@ Rank Official Party-State Chamber Trades Suspicious Rate
|
||||
|
||||
---
|
||||
|
||||
## **Deployment Checklist**
|
||||
## 🚀 **Deployment Checklist**
|
||||
|
||||
### **On Proxmox Container**
|
||||
|
||||
@@ -383,7 +383,7 @@ python scripts/generate_pattern_report.py --days 365
|
||||
|
||||
---
|
||||
|
||||
## **Documentation**
|
||||
## 📚 **Documentation**
|
||||
|
||||
- **`docs/11_live_market_monitoring.md`** - Deep dive into monitoring
|
||||
- **`LOCAL_TEST_GUIDE.md`** - Testing instructions
|
||||
@@ -392,23 +392,23 @@ python scripts/generate_pattern_report.py --days 365
|
||||
|
||||
---
|
||||
|
||||
## **Achievement Unlocked!**
|
||||
## 🎉 **Achievement Unlocked!**
|
||||
|
||||
**You now have a complete system that:**
|
||||
|
||||
Monitors real-time market activity
|
||||
Correlates trades to prior alerts
|
||||
Calculates timing advantage scores
|
||||
Identifies repeat offenders
|
||||
Ranks officials by suspicion
|
||||
Generates comprehensive reports
|
||||
93 tests confirming it works
|
||||
✅ Monitors real-time market activity
|
||||
✅ Correlates trades to prior alerts
|
||||
✅ Calculates timing advantage scores
|
||||
✅ Identifies repeat offenders
|
||||
✅ Ranks officials by suspicion
|
||||
✅ Generates comprehensive reports
|
||||
✅ 93 tests confirming it works
|
||||
|
||||
**This is a production-ready transparency and research tool!**
|
||||
**This is a production-ready transparency and research tool!** 🚀
|
||||
|
||||
---
|
||||
|
||||
## **Potential Future Enhancements**
|
||||
## 🔜 **Potential Future Enhancements**
|
||||
|
||||
### **Phase 4 Ideas (Optional)**
|
||||
- Email/SMS alerts for high-severity patterns
|
||||
@@ -420,6 +420,6 @@ python scripts/generate_pattern_report.py --days 365
|
||||
- Automated PDF reports
|
||||
- Historical performance tracking
|
||||
|
||||
**But the core system is COMPLETE and FUNCTIONAL now!**
|
||||
**But the core system is COMPLETE and FUNCTIONAL now!** ✅
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Offline Demo - Works Without Internet!
|
||||
|
||||
## Full System Working Without Network Access
|
||||
## ✅ Full System Working Without Network Access
|
||||
|
||||
Even though your environment doesn't have external internet, **everything works perfectly** using fixture files.
|
||||
|
||||
@@ -10,9 +10,9 @@ Even though your environment doesn't have external internet, **everything works
|
||||
python scripts/ingest_from_fixtures.py
|
||||
|
||||
# Output:
|
||||
# Officials created/updated: 4
|
||||
# Securities created/updated: 2
|
||||
# Trades ingested: 5
|
||||
# ✓ Officials created/updated: 4
|
||||
# ✓ Securities created/updated: 2
|
||||
# ✓ Trades ingested: 5
|
||||
#
|
||||
# Database totals:
|
||||
# Total officials: 4
|
||||
@@ -33,7 +33,7 @@ python scripts/ingest_from_fixtures.py
|
||||
- NVDA, MSFT, AAPL, TSLA, GOOGL tickers
|
||||
|
||||
2. **Offline Scripts**
|
||||
- `scripts/ingest_from_fixtures.py` - Ingest sample trades ( works now!)
|
||||
- `scripts/ingest_from_fixtures.py` - Ingest sample trades (✅ works now!)
|
||||
- `scripts/fetch_sample_prices.py` - Would need network (yfinance)
|
||||
|
||||
3. **28 Passing Tests** - All use mocks, no network required
|
||||
@@ -67,14 +67,14 @@ with SessionLocal() as session:
|
||||
|
||||
### What You Can Do Offline
|
||||
|
||||
**Run all tests**: `make test`
|
||||
**Ingest fixture data**: `python scripts/ingest_from_fixtures.py`
|
||||
**Query the database**: Use Python REPL or SQLite browser
|
||||
**Lint & format**: `make lint format`
|
||||
**Run migrations**: `make migrate`
|
||||
**Build analytics** (Phase 2): All math/ML works offline!
|
||||
✅ **Run all tests**: `make test`
|
||||
✅ **Ingest fixture data**: `python scripts/ingest_from_fixtures.py`
|
||||
✅ **Query the database**: Use Python REPL or SQLite browser
|
||||
✅ **Lint & format**: `make lint format`
|
||||
✅ **Run migrations**: `make migrate`
|
||||
✅ **Build analytics** (Phase 2): All math/ML works offline!
|
||||
|
||||
**Can't do (needs network)**:
|
||||
❌ **Can't do (needs network)**:
|
||||
- Fetch live congressional trades from House Stock Watcher
|
||||
- Fetch stock prices from yfinance
|
||||
- (But you can add more fixture files to simulate this!)
|
||||
@@ -108,9 +108,9 @@ You can expand the fixtures for offline development:
|
||||
## Summary
|
||||
|
||||
**The network error is not a problem!** The entire system is designed to work with:
|
||||
- Fixtures for development/testing
|
||||
- Real APIs for production (when network available)
|
||||
- Same code paths for both
|
||||
- ✅ Fixtures for development/testing
|
||||
- ✅ Real APIs for production (when network available)
|
||||
- ✅ Same code paths for both
|
||||
|
||||
This is **by design** - makes development fast and tests reliable!
|
||||
This is **by design** - makes development fast and tests reliable! 🚀
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Proxmox Quick Start
|
||||
# Proxmox Quick Start ⚡
|
||||
|
||||
**Got Proxmox? Deploy POTE in 5 minutes!**
|
||||
|
||||
@@ -17,7 +17,7 @@ su - poteapp
|
||||
cd pote && source venv/bin/activate
|
||||
python scripts/ingest_from_fixtures.py
|
||||
|
||||
# Done!
|
||||
# Done! ✅
|
||||
```
|
||||
|
||||
---
|
||||
@@ -81,8 +81,8 @@ source venv/bin/activate
|
||||
python scripts/ingest_from_fixtures.py
|
||||
|
||||
# Should see:
|
||||
# Officials created: 4
|
||||
# Trades ingested: 5
|
||||
# ✓ Officials created: 4
|
||||
# ✓ Trades ingested: 5
|
||||
```
|
||||
|
||||
### 5. Setup Cron Jobs
|
||||
@@ -96,7 +96,7 @@ crontab -e
|
||||
15 6 * * * cd /home/poteapp/pote && /home/poteapp/pote/venv/bin/python scripts/enrich_securities.py >> /home/poteapp/logs/enrich.log 2>&1
|
||||
```
|
||||
|
||||
### 6. Done!
|
||||
### 6. Done! 🎉
|
||||
|
||||
Your POTE instance is now running and will:
|
||||
- Fetch congressional trades daily at 6 AM
|
||||
@@ -107,12 +107,12 @@ Your POTE instance is now running and will:
|
||||
|
||||
## What You Get
|
||||
|
||||
**Full PostgreSQL database**
|
||||
**Automated daily updates** (via cron)
|
||||
**Isolated environment** (LXC container)
|
||||
**Easy backups** (Proxmox snapshots)
|
||||
**Low resource usage** (~500MB RAM)
|
||||
**Cost**: Just electricity (~$5-10/mo)
|
||||
✅ **Full PostgreSQL database**
|
||||
✅ **Automated daily updates** (via cron)
|
||||
✅ **Isolated environment** (LXC container)
|
||||
✅ **Easy backups** (Proxmox snapshots)
|
||||
✅ **Low resource usage** (~500MB RAM)
|
||||
✅ **Cost**: Just electricity (~$5-10/mo)
|
||||
|
||||
---
|
||||
|
||||
@@ -231,13 +231,13 @@ pip install -e .
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Container running
|
||||
2. POTE installed
|
||||
3. Data ingested
|
||||
4. ⏭ Setup Proxmox backups (Web UI → Datacenter → Backup)
|
||||
5. ⏭ Configure static IP (if needed)
|
||||
6. ⏭ Build Phase 2 analytics
|
||||
7. ⏭ Add FastAPI dashboard
|
||||
1. ✅ Container running
|
||||
2. ✅ POTE installed
|
||||
3. ✅ Data ingested
|
||||
4. ⏭️ Setup Proxmox backups (Web UI → Datacenter → Backup)
|
||||
5. ⏭️ Configure static IP (if needed)
|
||||
6. ⏭️ Build Phase 2 analytics
|
||||
7. ⏭️ Add FastAPI dashboard
|
||||
|
||||
---
|
||||
|
||||
@@ -264,10 +264,10 @@ pct restart 100
|
||||
|
||||
---
|
||||
|
||||
**Your Proxmox = Enterprise infrastructure at hobby prices!**
|
||||
**Your Proxmox = Enterprise infrastructure at hobby prices!** 🚀
|
||||
|
||||
Cost breakdown:
|
||||
- Cloud VPS: $20/mo
|
||||
- Your Proxmox: ~$10/mo (power)
|
||||
- **Savings: $120/year**
|
||||
- **Savings: $120/year** ✨
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# POTE Quick Start Guide
|
||||
|
||||
## Your System is Ready!
|
||||
## 🚀 Your System is Ready!
|
||||
|
||||
**Container IP**: Check with `ip addr show eth0 | grep "inet"`
|
||||
**Database**: PostgreSQL on port 5432
|
||||
**Username**: `poteuser`
|
||||
**Password**: `changeme123` ( change in production!)
|
||||
**Password**: `changeme123` (⚠️ change in production!)
|
||||
|
||||
---
|
||||
|
||||
## How to Use POTE
|
||||
## 📊 How to Use POTE
|
||||
|
||||
### Option 1: Command Line (SSH into container)
|
||||
|
||||
@@ -51,7 +51,7 @@ with engine.connect() as conn:
|
||||
|
||||
---
|
||||
|
||||
## Common Tasks
|
||||
## 🎯 Common Tasks
|
||||
|
||||
### 1. Check System Status
|
||||
|
||||
@@ -145,7 +145,7 @@ ORDER BY trade_count DESC;
|
||||
|
||||
---
|
||||
|
||||
## Example Workflows
|
||||
## 📈 Example Workflows
|
||||
|
||||
### Workflow 1: Daily Update
|
||||
|
||||
@@ -237,7 +237,7 @@ print(f"Exported {len(df)} trades to trades_export.csv")
|
||||
|
||||
---
|
||||
|
||||
## Maintenance
|
||||
## 🔧 Maintenance
|
||||
|
||||
### Update POTE Code
|
||||
|
||||
@@ -287,7 +287,7 @@ nano ~/pote/.env
|
||||
|
||||
---
|
||||
|
||||
## Access Methods Summary
|
||||
## 🌐 Access Methods Summary
|
||||
|
||||
| Method | From Where | Command |
|
||||
|--------|-----------|---------|
|
||||
@@ -298,22 +298,22 @@ nano ~/pote/.env
|
||||
|
||||
---
|
||||
|
||||
## What Data Do You Have?
|
||||
## 📚 What Data Do You Have?
|
||||
|
||||
Right now (Phase 1 complete):
|
||||
- **Congressional trading data** (from House Stock Watcher)
|
||||
- **Security information** (tickers, names, sectors)
|
||||
- **Historical prices** (OHLCV data from yfinance)
|
||||
- **Official profiles** (name, party, chamber, state)
|
||||
- ✅ **Congressional trading data** (from House Stock Watcher)
|
||||
- ✅ **Security information** (tickers, names, sectors)
|
||||
- ✅ **Historical prices** (OHLCV data from yfinance)
|
||||
- ✅ **Official profiles** (name, party, chamber, state)
|
||||
|
||||
Coming next (Phase 2):
|
||||
- **Abnormal return calculations**
|
||||
- **Behavioral clustering**
|
||||
- **Research signals** (follow_research, avoid_risk, watch)
|
||||
- 📊 **Abnormal return calculations**
|
||||
- 🤖 **Behavioral clustering**
|
||||
- 🚨 **Research signals** (follow_research, avoid_risk, watch)
|
||||
|
||||
---
|
||||
|
||||
## Learning SQL for POTE
|
||||
## 🎓 Learning SQL for POTE
|
||||
|
||||
### Count Records
|
||||
```sql
|
||||
@@ -349,7 +349,7 @@ GROUP BY o.party;
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
## ❓ Troubleshooting
|
||||
|
||||
### Can't connect remotely?
|
||||
```bash
|
||||
@@ -380,12 +380,12 @@ pip install -e .
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
## 🚀 Next Steps
|
||||
|
||||
1. **Populate with real data**: Run `fetch_congressional_trades.py` regularly
|
||||
2. **Set up cron job** for automatic daily updates
|
||||
3. **Build analytics** (Phase 2) - abnormal returns, signals
|
||||
4. **Create dashboard** (Phase 3) - web interface for exploration
|
||||
|
||||
Ready to build Phase 2 analytics? Just ask!
|
||||
Ready to build Phase 2 analytics? Just ask! 📈
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# POTE Quick Setup Card
|
||||
# 🚀 POTE Quick Setup Card
|
||||
|
||||
## Your Configuration
|
||||
## 📍 Your Configuration
|
||||
|
||||
**Email Server:** `mail.levkin.ca`
|
||||
**Email Account:** `test@levkin.ca`
|
||||
**Database:** PostgreSQL (configured)
|
||||
**Status:** Ready for deployment
|
||||
**Status:** ✅ Ready for deployment
|
||||
|
||||
---
|
||||
|
||||
## 3-Step Setup
|
||||
## ⚡ 3-Step Setup
|
||||
|
||||
### Step 1: Add Your Password (30 seconds)
|
||||
|
||||
@@ -27,7 +27,7 @@ source venv/bin/activate
|
||||
python scripts/send_daily_report.py --to test@levkin.ca --test-smtp
|
||||
```
|
||||
|
||||
**Check test@levkin.ca inbox** - you should receive a test email!
|
||||
✅ **Check test@levkin.ca inbox** - you should receive a test email!
|
||||
|
||||
### Step 3: Automate (2 minutes)
|
||||
|
||||
@@ -37,13 +37,13 @@ python scripts/send_daily_report.py --to test@levkin.ca --test-smtp
|
||||
# Choose time: 6 AM (recommended)
|
||||
```
|
||||
|
||||
**Done!** You'll now receive:
|
||||
**Done!** 🎉 You'll now receive:
|
||||
- Daily reports at 6 AM
|
||||
- Weekly reports on Sundays
|
||||
|
||||
---
|
||||
|
||||
## Deployment to Proxmox (5 minutes)
|
||||
## 📦 Deployment to Proxmox (5 minutes)
|
||||
|
||||
### On Proxmox Host:
|
||||
|
||||
@@ -79,7 +79,7 @@ bash scripts/proxmox_setup.sh
|
||||
|
||||
---
|
||||
|
||||
## Quick Commands
|
||||
## 🔍 Quick Commands
|
||||
|
||||
### On Deployed Server (SSH)
|
||||
|
||||
@@ -105,7 +105,7 @@ ls -lh ~/logs/*.txt
|
||||
|
||||
---
|
||||
|
||||
## Email Configuration (.env)
|
||||
## 📧 Email Configuration (.env)
|
||||
|
||||
```env
|
||||
SMTP_HOST=mail.levkin.ca
|
||||
@@ -123,27 +123,27 @@ REPORT_RECIPIENTS=test@levkin.ca,user2@example.com,user3@example.com
|
||||
|
||||
---
|
||||
|
||||
## What You'll Receive
|
||||
## 📊 What You'll Receive
|
||||
|
||||
### Daily Report (6 AM)
|
||||
```
|
||||
New congressional trades
|
||||
Market alerts (unusual activity)
|
||||
Suspicious timing detections
|
||||
Summary statistics
|
||||
✅ New congressional trades
|
||||
✅ Market alerts (unusual activity)
|
||||
✅ Suspicious timing detections
|
||||
✅ Summary statistics
|
||||
```
|
||||
|
||||
### Weekly Report (Sunday 8 AM)
|
||||
```
|
||||
Most active officials
|
||||
Most traded securities
|
||||
Repeat offenders
|
||||
Pattern analysis
|
||||
✅ Most active officials
|
||||
✅ Most traded securities
|
||||
✅ Repeat offenders
|
||||
✅ Pattern analysis
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
## 🔧 Troubleshooting
|
||||
|
||||
### Email Not Working?
|
||||
|
||||
@@ -191,19 +191,19 @@ tail -50 ~/logs/daily_run.log
|
||||
|
||||
---
|
||||
|
||||
## Documentation
|
||||
## 📚 Documentation
|
||||
|
||||
| Document | Purpose |
|
||||
|----------|---------|
|
||||
| **[EMAIL_SETUP.md](EMAIL_SETUP.md)** | Your levkin.ca setup guide |
|
||||
| **[DEPLOYMENT_AND_AUTOMATION.md](DEPLOYMENT_AND_AUTOMATION.md)** | Answers all questions |
|
||||
| **[EMAIL_SETUP.md](EMAIL_SETUP.md)** | ⭐ Your levkin.ca setup guide |
|
||||
| **[DEPLOYMENT_AND_AUTOMATION.md](DEPLOYMENT_AND_AUTOMATION.md)** | ⭐ Answers all questions |
|
||||
| **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** | Quick automation guide |
|
||||
| **[PROXMOX_QUICKSTART.md](PROXMOX_QUICKSTART.md)** | Proxmox deployment |
|
||||
| **[QUICKSTART.md](QUICKSTART.md)** | Usage guide |
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
## ✅ Checklist
|
||||
|
||||
**Local Development:**
|
||||
- [ ] `.env` file created with password
|
||||
@@ -227,18 +227,18 @@ tail -50 ~/logs/daily_run.log
|
||||
|
||||
---
|
||||
|
||||
## Your Current Status
|
||||
## 🎯 Your Current Status
|
||||
|
||||
**Code:** Complete (93 tests passing)
|
||||
**Monitoring:** 3-phase system operational
|
||||
**CI/CD:** Pipeline ready (.github/workflows/ci.yml)
|
||||
**Email:** Configured for test@levkin.ca
|
||||
✅ **Code:** Complete (93 tests passing)
|
||||
✅ **Monitoring:** 3-phase system operational
|
||||
✅ **CI/CD:** Pipeline ready (.github/workflows/ci.yml)
|
||||
✅ **Email:** Configured for test@levkin.ca
|
||||
⏳ **Deployment:** Ready to deploy to Proxmox
|
||||
⏳ **Automation:** Ready to set up with `setup_cron.sh`
|
||||
|
||||
---
|
||||
|
||||
## Next Action
|
||||
## 🚀 Next Action
|
||||
|
||||
**Right now (local testing):**
|
||||
```bash
|
||||
@@ -255,9 +255,9 @@ cd ~/pote
|
||||
./scripts/setup_cron.sh
|
||||
```
|
||||
|
||||
**That's it! **
|
||||
**That's it! 🎉**
|
||||
|
||||
---
|
||||
|
||||
**Everything is ready - just add your password and test!**
|
||||
**Everything is ready - just add your password and test!** 📧
|
||||
|
||||
@@ -1,67 +1,202 @@
|
||||
# POTE (Public Officials Trading Explorer)
|
||||
# POTE – Public Officials Trading Explorer
|
||||
|
||||
Research tool for tracking publicly disclosed stock trades by government
|
||||
officials (starting with U.S. Congress). Computes descriptive metrics and
|
||||
risk/ethics flags from lawfully available public data.
|
||||
**Research-only tool for tracking and analyzing public stock trades by government officials.**
|
||||
|
||||
Not investment advice. Not for live trading. Public disclosures only; data
|
||||
may be delayed or incomplete. No claims about inside information.
|
||||
⚠️ **Important**: This project is for personal research and transparency analysis only. It is **NOT** for investment advice or live trading.
|
||||
|
||||
Status: active. Homelab LXC deploy documented under `docs/`.
|
||||
## What is this?
|
||||
|
||||
POTE tracks stock trading activity of government officials (starting with U.S. Congress) using lawfully available public data sources. It computes research metrics, descriptive signals, and risk/ethics flags to help understand trading patterns.
|
||||
|
||||
## Key constraints
|
||||
|
||||
- **Public data only**: House Stock Watcher (free!), yfinance (free!), QuiverQuant/FMP (optional)
|
||||
- **Research framing**: All outputs are descriptive analytics, not trading recommendations
|
||||
- **No inside information claims**: We use public disclosures that may be delayed or incomplete
|
||||
|
||||
## Current Status
|
||||
|
||||
✅ **PR1 Complete**: Project scaffold, DB models, price loader
|
||||
✅ **PR2 Complete**: Congressional trade ingestion (House Stock Watcher)
|
||||
✅ **PR3 Complete**: Security enrichment + deployment infrastructure
|
||||
✅ **PR4 Complete**: Phase 2 analytics - returns, benchmarks, performance metrics
|
||||
**45+ passing tests, 88%+ coverage**
|
||||
|
||||
## Quick start
|
||||
|
||||
**🚀 Already deployed?** See **[QUICKSTART.md](QUICKSTART.md)** for full usage guide!
|
||||
|
||||
**📦 Deploying?** See **[PROXMOX_QUICKSTART.md](PROXMOX_QUICKSTART.md)** for Proxmox LXC deployment (recommended).
|
||||
|
||||
**📧 Want automated reports?** See **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** for email reporting setup!
|
||||
|
||||
### Local Development
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd pote # or POTE
|
||||
# Install
|
||||
git clone <your-repo>
|
||||
cd pote
|
||||
make install
|
||||
source venv/bin/activate
|
||||
|
||||
# Run migrations
|
||||
make migrate
|
||||
python scripts/ingest_from_fixtures.py # offline sample
|
||||
make test
|
||||
```
|
||||
|
||||
With network:
|
||||
# Ingest sample data (offline, for testing)
|
||||
python scripts/ingest_from_fixtures.py
|
||||
|
||||
```bash
|
||||
# Enrich securities with company info
|
||||
python scripts/enrich_securities.py
|
||||
|
||||
# With internet:
|
||||
python scripts/fetch_congressional_trades.py
|
||||
python scripts/fetch_sample_prices.py
|
||||
|
||||
# Run tests
|
||||
make test
|
||||
|
||||
# Lint & format
|
||||
make lint format
|
||||
```
|
||||
|
||||
Deploy: `bash scripts/proxmox_setup.sh` or `docker-compose up -d`.
|
||||
Ops handoff: [docs/HANDOFF-2026-05-27.md](docs/HANDOFF-2026-05-27.md).
|
||||
|
||||
## Useful commands
|
||||
|
||||
### Production Deployment
|
||||
```bash
|
||||
# Proxmox LXC (Recommended - 5 minutes)
|
||||
bash scripts/proxmox_setup.sh
|
||||
|
||||
# Docker
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Language**: Python 3.10+
|
||||
- **Database**: PostgreSQL or SQLite (dev)
|
||||
- **Data**: House Stock Watcher (free!), yfinance (free!), QuiverQuant/FMP (optional)
|
||||
- **Libraries**: SQLAlchemy, Alembic, pandas, numpy, httpx, yfinance, scikit-learn
|
||||
- **Testing**: pytest (28 tests, 87%+ coverage)
|
||||
|
||||
## Documentation
|
||||
|
||||
**Getting Started**:
|
||||
- [`README.md`](README.md) – This file
|
||||
- [`QUICKSTART.md`](QUICKSTART.md) – ⭐ **How to use your deployed POTE instance**
|
||||
- [`STATUS.md`](STATUS.md) – Current project status
|
||||
- [`FREE_TESTING_QUICKSTART.md`](FREE_TESTING_QUICKSTART.md) – Test for $0
|
||||
- [`OFFLINE_DEMO.md`](OFFLINE_DEMO.md) – Works without internet!
|
||||
|
||||
**Deployment**:
|
||||
- [`PROXMOX_QUICKSTART.md`](PROXMOX_QUICKSTART.md) – ⭐ **Proxmox quick deployment (5 min)**
|
||||
- [`AUTOMATION_QUICKSTART.md`](AUTOMATION_QUICKSTART.md) – ⭐ **Automated reporting setup (5 min)**
|
||||
- [`docs/07_deployment.md`](docs/07_deployment.md) – Full deployment guide (all platforms)
|
||||
- [`docs/08_proxmox_deployment.md`](docs/08_proxmox_deployment.md) – Proxmox detailed guide
|
||||
- [`docs/12_automation_and_reporting.md`](docs/12_automation_and_reporting.md) – Automation & CI/CD guide
|
||||
- [`Dockerfile`](Dockerfile) + [`docker-compose.yml`](docker-compose.yml) – Docker setup
|
||||
|
||||
**Technical**:
|
||||
- [`docs/00_mvp.md`](docs/00_mvp.md) – MVP roadmap
|
||||
- [`docs/01_architecture.md`](docs/01_architecture.md) – Architecture
|
||||
- [`docs/02_data_model.md`](docs/02_data_model.md) – Database schema
|
||||
- [`docs/03_data_sources.md`](docs/03_data_sources.md) – Data sources
|
||||
- [`docs/04_safety_ethics.md`](docs/04_safety_ethics.md) – Research-only guardrails
|
||||
- [`docs/05_dev_setup.md`](docs/05_dev_setup.md) – Dev conventions
|
||||
- [`docs/06_free_testing_data.md`](docs/06_free_testing_data.md) – Testing strategies
|
||||
|
||||
**PR Summaries**:
|
||||
- [`docs/PR1_SUMMARY.md`](docs/PR1_SUMMARY.md) – Scaffold + price loader
|
||||
- [`docs/PR2_SUMMARY.md`](docs/PR2_SUMMARY.md) – Congressional trades
|
||||
- [`docs/PR3_SUMMARY.md`](docs/PR3_SUMMARY.md) – Enrichment + deployment
|
||||
- [`docs/PR4_SUMMARY.md`](docs/PR4_SUMMARY.md) – ⭐ **Analytics foundation (returns, benchmarks, metrics)**
|
||||
|
||||
## What's Working Now
|
||||
|
||||
- ✅ SQLAlchemy models for officials, securities, trades, prices
|
||||
- ✅ Alembic migrations
|
||||
- ✅ Price loader with yfinance (idempotent, upsert)
|
||||
- ✅ Congressional trade ingestion from House Stock Watcher (FREE!)
|
||||
- ✅ Security enrichment (company names, sectors, industries)
|
||||
- ✅ ETL to populate officials & trades tables
|
||||
- ✅ Docker + deployment infrastructure
|
||||
- ✅ 93 passing tests with 88%+ coverage
|
||||
- ✅ Linting (ruff + mypy) all green
|
||||
- ✅ Works 100% offline with fixtures
|
||||
- ✅ Real-time market monitoring & alert system
|
||||
- ✅ Disclosure timing correlation engine
|
||||
- ✅ Pattern detection & comparative analysis
|
||||
- ✅ Automated email reporting (daily/weekly)
|
||||
- ✅ CI/CD pipeline (GitHub/Gitea Actions)
|
||||
|
||||
## What You Can Do Now
|
||||
|
||||
### Analyze Performance
|
||||
```bash
|
||||
# Analyze specific official
|
||||
python scripts/analyze_official.py "Nancy Pelosi" --window 90
|
||||
|
||||
# System-wide analysis
|
||||
python scripts/calculate_all_returns.py
|
||||
```
|
||||
|
||||
### Market Monitoring
|
||||
```bash
|
||||
# Run market scan
|
||||
python scripts/monitor_market.py --scan
|
||||
|
||||
# Analyze timing of recent disclosures
|
||||
python scripts/analyze_disclosure_timing.py --recent 7
|
||||
|
||||
# Generate pattern report
|
||||
python scripts/generate_pattern_report.py --days 365
|
||||
```
|
||||
|
||||
### Automated Reporting
|
||||
```bash
|
||||
# Set up daily/weekly email reports (5 minutes!)
|
||||
./scripts/setup_cron.sh
|
||||
|
||||
# Send manual report
|
||||
python scripts/send_daily_report.py --to your@email.com
|
||||
```
|
||||
|
||||
## Stack
|
||||
### Add More Data
|
||||
```bash
|
||||
# Manual entry
|
||||
python scripts/add_custom_trades.py
|
||||
|
||||
Python 3.10+, SQLAlchemy, Alembic, pandas/numpy, httpx, yfinance.
|
||||
PostgreSQL or SQLite for local. pytest + ruff + mypy.
|
||||
# CSV import
|
||||
python scripts/scrape_alternative_sources.py import trades.csv
|
||||
```
|
||||
|
||||
Data sources: House Stock Watcher, yfinance; QuiverQuant/FMP optional.
|
||||
## System Architecture
|
||||
|
||||
## Docs
|
||||
POTE now includes a complete 3-phase monitoring system:
|
||||
|
||||
| Doc | Purpose |
|
||||
|-----|---------|
|
||||
| [docs/QUICKSTART.md](docs/QUICKSTART.md) | Using a deployed instance |
|
||||
| [docs/PROXMOX_QUICKSTART.md](docs/PROXMOX_QUICKSTART.md) | Proxmox LXC |
|
||||
| [docs/AUTOMATION_QUICKSTART.md](docs/AUTOMATION_QUICKSTART.md) | Email reports |
|
||||
| [docs/01_architecture.md](docs/01_architecture.md) | Architecture |
|
||||
| [docs/02_data_model.md](docs/02_data_model.md) | Schema |
|
||||
| [docs/04_safety_ethics.md](docs/04_safety_ethics.md) | Research guardrails |
|
||||
| [docs/07_deployment.md](docs/07_deployment.md) | Full deploy |
|
||||
| [docs/00_mvp.md](docs/00_mvp.md) | Roadmap |
|
||||
**Phase 1: Real-Time Market Monitoring**
|
||||
- Tracks ~50 most-traded congressional stocks
|
||||
- Detects unusual volume, price spikes, volatility
|
||||
- Logs all alerts with timestamps and severity
|
||||
|
||||
PR writeups live under `docs/archive/` when moved.
|
||||
**Phase 2: Disclosure Correlation**
|
||||
- Matches trades with prior market alerts (30-45 day lookback)
|
||||
- Calculates "timing advantage score" (0-100)
|
||||
- Identifies suspicious timing patterns
|
||||
|
||||
## License
|
||||
**Phase 3: Pattern Detection**
|
||||
- Ranks officials by consistent suspicious timing
|
||||
- Analyzes by ticker, sector, and political party
|
||||
- Generates comprehensive reports
|
||||
|
||||
MIT for research/educational use. Not investment advice.
|
||||
**Full Documentation**: See [`MONITORING_SYSTEM_COMPLETE.md`](MONITORING_SYSTEM_COMPLETE.md)
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [ ] Signals: "follow_research", "avoid_risk", "watch" with confidence scores
|
||||
- [ ] Clustering: group officials by trading behavior patterns
|
||||
- [ ] API: FastAPI backend for queries
|
||||
- [ ] Dashboard: React/Streamlit visualization
|
||||
|
||||
See [`docs/00_mvp.md`](docs/00_mvp.md) for the full roadmap.
|
||||
|
||||
---
|
||||
|
||||
**License**: MIT (for research/educational use only)
|
||||
**Disclaimer**: Not investment advice. Use public data only. No claims about inside information.
|
||||
|
||||
+27
-27
@@ -3,25 +3,25 @@
|
||||
**Last Updated**: 2025-12-14
|
||||
**Version**: Phase 1 Complete (PR1 + PR2)
|
||||
|
||||
## What's Working Now
|
||||
## 🎉 What's Working Now
|
||||
|
||||
### Data Ingestion (FREE!)
|
||||
**Congressional Trades**: Live ingestion from House Stock Watcher
|
||||
**Stock Prices**: Daily OHLCV from yfinance
|
||||
**Officials**: Auto-populated from trade disclosures
|
||||
**Securities**: Auto-created, ready for enrichment
|
||||
✅ **Congressional Trades**: Live ingestion from House Stock Watcher
|
||||
✅ **Stock Prices**: Daily OHLCV from yfinance
|
||||
✅ **Officials**: Auto-populated from trade disclosures
|
||||
✅ **Securities**: Auto-created, ready for enrichment
|
||||
|
||||
### Database
|
||||
**Schema**: Normalized (officials, securities, trades, prices, metrics stubs)
|
||||
**Migrations**: Alembic configured and applied
|
||||
**DB**: SQLite for dev, PostgreSQL-ready
|
||||
✅ **Schema**: Normalized (officials, securities, trades, prices, metrics stubs)
|
||||
✅ **Migrations**: Alembic configured and applied
|
||||
✅ **DB**: SQLite for dev, PostgreSQL-ready
|
||||
|
||||
### Code Quality
|
||||
**Tests**: 28 passing (86% coverage)
|
||||
**Linting**: ruff + mypy all green
|
||||
**Format**: black applied consistently
|
||||
✅ **Tests**: 28 passing (86% coverage)
|
||||
✅ **Linting**: ruff + mypy all green
|
||||
✅ **Format**: black applied consistently
|
||||
|
||||
## Current Stats
|
||||
## 📊 Current Stats
|
||||
|
||||
```bash
|
||||
# Test Suite
|
||||
@@ -43,7 +43,7 @@ All free/open-source:
|
||||
- pytest (testing)
|
||||
```
|
||||
|
||||
## Quick Commands
|
||||
## 🚀 Quick Commands
|
||||
|
||||
### Fetch Live Data (FREE!)
|
||||
```bash
|
||||
@@ -75,7 +75,7 @@ make format # Format with black
|
||||
make migrate # Run Alembic migrations
|
||||
```
|
||||
|
||||
## Deployment
|
||||
## 🏠 Deployment
|
||||
|
||||
**Your Proxmox?** Perfect! See [`docs/08_proxmox_deployment.md`](docs/08_proxmox_deployment.md) for:
|
||||
- LXC container setup (lightweight, recommended)
|
||||
@@ -90,7 +90,7 @@ Other options in [`docs/07_deployment.md`](docs/07_deployment.md):
|
||||
- Railway/Fly.io - $5-15/mo
|
||||
- AWS/GCP - $20-50/mo
|
||||
|
||||
## Project Structure
|
||||
## 📂 Project Structure
|
||||
|
||||
```
|
||||
pote/
|
||||
@@ -138,7 +138,7 @@ pote/
|
||||
└── fetch_sample_prices.py # Live price fetch
|
||||
```
|
||||
|
||||
## Cost Breakdown
|
||||
## 💰 Cost Breakdown
|
||||
|
||||
| Component | Cost | Notes |
|
||||
|-----------|------|-------|
|
||||
@@ -154,7 +154,7 @@ Optional paid upgrades (NOT needed):
|
||||
- Financial Modeling Prep: $15/mo (250 calls/day free tier available)
|
||||
- PostgreSQL hosting: $7+/mo (only if deploying)
|
||||
|
||||
## Completed PRs
|
||||
## ✅ Completed PRs
|
||||
|
||||
### PR1: Project Scaffold + Price Loader
|
||||
- [x] Project structure (`src/`, `tests/`, docs)
|
||||
@@ -176,7 +176,7 @@ Optional paid upgrades (NOT needed):
|
||||
|
||||
**See**: [`docs/PR2_SUMMARY.md`](docs/PR2_SUMMARY.md)
|
||||
|
||||
## Next Steps (Phase 2 - Analytics)
|
||||
## 📋 Next Steps (Phase 2 - Analytics)
|
||||
|
||||
### PR3: Security Enrichment
|
||||
- [ ] Enrich securities table with yfinance (names, sectors, exchanges)
|
||||
@@ -206,20 +206,20 @@ Optional paid upgrades (NOT needed):
|
||||
|
||||
**See**: [`docs/00_mvp.md`](docs/00_mvp.md) for full roadmap
|
||||
|
||||
## Research-Only Reminder
|
||||
## 🔬 Research-Only Reminder
|
||||
|
||||
**This tool is for private research and transparency analysis only.**
|
||||
|
||||
- Not investment advice
|
||||
- Not a trading system
|
||||
- No claims about inside information
|
||||
- Public data only
|
||||
- Descriptive analytics
|
||||
- Research transparency
|
||||
- ❌ Not investment advice
|
||||
- ❌ Not a trading system
|
||||
- ❌ No claims about inside information
|
||||
- ✅ Public data only
|
||||
- ✅ Descriptive analytics
|
||||
- ✅ Research transparency
|
||||
|
||||
See [`docs/04_safety_ethics.md`](docs/04_safety_ethics.md) for guardrails.
|
||||
|
||||
## Contributing
|
||||
## 🤝 Contributing
|
||||
|
||||
This is a personal research project, but if you want to use it:
|
||||
|
||||
@@ -229,7 +229,7 @@ This is a personal research project, but if you want to use it:
|
||||
4. `python scripts/fetch_congressional_trades.py --days 7`
|
||||
5. Start exploring!
|
||||
|
||||
## License
|
||||
## 📄 License
|
||||
|
||||
MIT License (for research/educational use only)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# POTE Watchlist & Trading Reports
|
||||
|
||||
## Get Trading Reports 1 Hour Before Market Close
|
||||
## 🎯 Get Trading Reports 1 Hour Before Market Close
|
||||
|
||||
### Quick Setup
|
||||
|
||||
@@ -25,7 +25,7 @@ crontab -e
|
||||
|
||||
---
|
||||
|
||||
## Watchlist System
|
||||
## 📋 Watchlist System
|
||||
|
||||
### Who's on the Default Watchlist?
|
||||
|
||||
@@ -48,7 +48,7 @@ crontab -e
|
||||
|
||||
---
|
||||
|
||||
## Managing Your Watchlist
|
||||
## 🔧 Managing Your Watchlist
|
||||
|
||||
### View Current Watchlist
|
||||
|
||||
@@ -104,7 +104,7 @@ This fetches all 535 members of Congress (100 Senate + 435 House).
|
||||
|
||||
---
|
||||
|
||||
## Generating Reports
|
||||
## 📊 Generating Reports
|
||||
|
||||
### Manual Report Generation
|
||||
|
||||
@@ -135,24 +135,24 @@ python scripts/generate_trading_report.py --format json --output report.json
|
||||
================================================================================
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Nancy Pelosi (D-CA, House)
|
||||
👤 Nancy Pelosi (D-CA, House)
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Side Ticker Company Sector Value Trade Date Filed
|
||||
-------- ------ -------------------------- ---------- ------------------- ---------- ----------
|
||||
BUY NVDA NVIDIA Corporation Technology $15,001 - $50,000 2024-11-15 2024-12-01
|
||||
SELL MSFT Microsoft Corporation Technology $50,001 - $100,000 2024-11-20 2024-12-01
|
||||
🟢 BUY NVDA NVIDIA Corporation Technology $15,001 - $50,000 2024-11-15 2024-12-01
|
||||
🔴 SELL MSFT Microsoft Corporation Technology $50,001 - $100,000 2024-11-20 2024-12-01
|
||||
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Tommy Tuberville (R-AL, Senate)
|
||||
👤 Tommy Tuberville (R-AL, Senate)
|
||||
────────────────────────────────────────────────────────────────────────────────
|
||||
Side Ticker Company Sector Value Trade Date Filed
|
||||
-------- ------ -------------------------- ---------- ------------------- ---------- ----------
|
||||
BUY SPY SPDR S&P 500 ETF Financial $100,001 - $250,000 2024-11-18 2024-12-02
|
||||
BUY AAPL Apple Inc. Technology $50,001 - $100,000 2024-11-22 2024-12-02
|
||||
SELL TSLA Tesla, Inc. Automotive $15,001 - $50,000 2024-11-25 2024-12-02
|
||||
🟢 BUY SPY SPDR S&P 500 ETF Financial $100,001 - $250,000 2024-11-18 2024-12-02
|
||||
🟢 BUY AAPL Apple Inc. Technology $50,001 - $100,000 2024-11-22 2024-12-02
|
||||
🔴 SELL TSLA Tesla, Inc. Automotive $15,001 - $50,000 2024-11-25 2024-12-02
|
||||
|
||||
================================================================================
|
||||
SUMMARY
|
||||
📊 SUMMARY
|
||||
================================================================================
|
||||
|
||||
Total Trades: 5
|
||||
@@ -223,7 +223,7 @@ Runs at 8 AM and 3 PM daily (weekdays).
|
||||
|
||||
---
|
||||
|
||||
## Email Reports (Optional)
|
||||
## 📧 Email Reports (Optional)
|
||||
|
||||
### Setup Email Notifications
|
||||
|
||||
@@ -256,7 +256,7 @@ python scripts/send_email.py /tmp/report.html your-email@example.com
|
||||
|
||||
---
|
||||
|
||||
## Typical Workflow
|
||||
## 🎯 Typical Workflow
|
||||
|
||||
### Daily Routine (3 PM ET)
|
||||
|
||||
@@ -282,7 +282,7 @@ python scripts/send_email.py /tmp/report.html your-email@example.com
|
||||
|
||||
---
|
||||
|
||||
## Finding More Officials
|
||||
## 🔍 Finding More Officials
|
||||
|
||||
### Public Resources
|
||||
|
||||
@@ -318,7 +318,7 @@ Add committee members to your watchlist.
|
||||
|
||||
---
|
||||
|
||||
## Example Cron Setup
|
||||
## 📈 Example Cron Setup
|
||||
|
||||
```bash
|
||||
# Edit crontab
|
||||
@@ -341,7 +341,7 @@ This gives you:
|
||||
|
||||
---
|
||||
|
||||
## Quick Start Summary
|
||||
## 🚀 Quick Start Summary
|
||||
|
||||
```bash
|
||||
# 1. Create watchlist
|
||||
@@ -363,7 +363,7 @@ cat reports/trading_report_$(date +%Y%m%d).txt
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
## ❓ FAQ
|
||||
|
||||
**Q: Why are all the trades old (30-45 days)?**
|
||||
**A:** Federal law (STOCK Act) gives Congress 30-45 days to file. This is normal.
|
||||
@@ -6,7 +6,7 @@ Great question! Here are multiple strategies for testing the full pipeline **wit
|
||||
|
||||
---
|
||||
|
||||
## Strategy 1: Mock/Fixture Data (Current Approach )
|
||||
## Strategy 1: Mock/Fixture Data (Current Approach ✅)
|
||||
|
||||
**What we already have:**
|
||||
- `tests/conftest.py` creates in-memory SQLite DB with sample officials, securities, trades
|
||||
@@ -99,7 +99,7 @@ def test_etl_with_real_data():
|
||||
|
||||
---
|
||||
|
||||
## Strategy 4: Hybrid Testing (Recommended )
|
||||
## Strategy 4: Hybrid Testing (Recommended 🌟)
|
||||
|
||||
**Combine all strategies**:
|
||||
|
||||
@@ -164,7 +164,7 @@ def test_etl_with_real_data():
|
||||
|
||||
| Source | Free Tier | Paid Tier | Best For |
|
||||
|--------|-----------|-----------|----------|
|
||||
| **yfinance** | Unlimited | N/A | Prices (already working ) |
|
||||
| **yfinance** | Unlimited | N/A | Prices (already working ✅) |
|
||||
| **House Stock Watcher** | Unlimited scraping | N/A | Free trades (best option) |
|
||||
| **Quiver Free** | 500 calls/mo | $30/mo (5k calls) | Testing, not production |
|
||||
| **FMP Free** | 250 calls/day | $15/mo | Alternative for trades |
|
||||
@@ -222,5 +222,5 @@ class HouseWatcherClient:
|
||||
}
|
||||
```
|
||||
|
||||
Let me know if you want me to implement this scraper now for PR2!
|
||||
Let me know if you want me to implement this scraper now for PR2! 🚀
|
||||
|
||||
|
||||
@@ -4,14 +4,14 @@
|
||||
|
||||
POTE can be deployed in several ways depending on your needs:
|
||||
|
||||
1. **Local Development** (SQLite) - What you have now
|
||||
1. **Local Development** (SQLite) - What you have now ✅
|
||||
2. **Single Server** (PostgreSQL + cron jobs)
|
||||
3. **Docker** (Containerized, easy to move)
|
||||
4. **Cloud** (AWS/GCP/Azure with managed DB)
|
||||
|
||||
---
|
||||
|
||||
## Option 1: Local Development (Current Setup)
|
||||
## Option 1: Local Development (Current Setup) ✅
|
||||
|
||||
**You're already running this!**
|
||||
|
||||
|
||||
@@ -2,17 +2,17 @@
|
||||
|
||||
## Why Proxmox is Perfect for POTE
|
||||
|
||||
**Full control** - Your hardware, your rules
|
||||
**No monthly costs** - Just electricity
|
||||
**Isolated VMs/LXC** - Clean environments
|
||||
**Snapshots** - Easy rollback if needed
|
||||
**Resource efficient** - Run alongside other services
|
||||
✅ **Full control** - Your hardware, your rules
|
||||
✅ **No monthly costs** - Just electricity
|
||||
✅ **Isolated VMs/LXC** - Clean environments
|
||||
✅ **Snapshots** - Easy rollback if needed
|
||||
✅ **Resource efficient** - Run alongside other services
|
||||
|
||||
---
|
||||
|
||||
## Deployment Options on Proxmox
|
||||
|
||||
### Option 1: LXC Container (Recommended)
|
||||
### Option 1: LXC Container (Recommended) ⭐
|
||||
|
||||
**Pros**: Lightweight, fast, efficient resource usage
|
||||
**Cons**: Linux only (fine for POTE)
|
||||
@@ -501,14 +501,14 @@ vs.
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Create LXC container
|
||||
2. Install dependencies
|
||||
3. Setup PostgreSQL
|
||||
4. Deploy POTE
|
||||
5. Configure cron jobs
|
||||
6. Setup backups
|
||||
7. ⏭ Build Phase 2 (Analytics)
|
||||
8. ⏭ Add FastAPI dashboard (optional)
|
||||
1. ✅ Create LXC container
|
||||
2. ✅ Install dependencies
|
||||
3. ✅ Setup PostgreSQL
|
||||
4. ✅ Deploy POTE
|
||||
5. ✅ Configure cron jobs
|
||||
6. ✅ Setup backups
|
||||
7. ⏭️ Build Phase 2 (Analytics)
|
||||
8. ⏭️ Add FastAPI dashboard (optional)
|
||||
|
||||
---
|
||||
|
||||
@@ -583,7 +583,7 @@ EOF
|
||||
sudo -u poteapp mkdir -p /home/poteapp/logs
|
||||
|
||||
echo ""
|
||||
echo " Setup complete!"
|
||||
echo "✅ Setup complete!"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. su - poteapp"
|
||||
@@ -600,5 +600,5 @@ chmod +x proxmox_setup.sh
|
||||
|
||||
---
|
||||
|
||||
**Your Proxmox setup gives you enterprise-grade infrastructure at hobby costs!**
|
||||
**Your Proxmox setup gives you enterprise-grade infrastructure at hobby costs!** 🚀
|
||||
|
||||
|
||||
@@ -137,13 +137,13 @@ python scripts/fetch_sample_prices.py
|
||||
## Data Sources
|
||||
|
||||
### Currently Working:
|
||||
- yfinance (prices, company info)
|
||||
- Manual entry
|
||||
- CSV import
|
||||
- Fixture files (testing)
|
||||
- ✅ yfinance (prices, company info)
|
||||
- ✅ Manual entry
|
||||
- ✅ CSV import
|
||||
- ✅ Fixture files (testing)
|
||||
|
||||
### Currently Down:
|
||||
- House Stock Watcher API (domain issues)
|
||||
- ❌ House Stock Watcher API (domain issues)
|
||||
|
||||
### Future Options:
|
||||
- QuiverQuant (requires $30/month subscription)
|
||||
|
||||
+19
-19
@@ -8,10 +8,10 @@
|
||||
### **Reality Check: No Real-Time Data Exists**
|
||||
|
||||
**Federal Law (STOCK Act):**
|
||||
- Congress members have **30-45 days** to disclose trades
|
||||
- Disclosures are filed as **Periodic Transaction Reports (PTRs)**
|
||||
- Public databases update **after** filing (usually next day)
|
||||
- **No real-time feed exists by design**
|
||||
- 📅 Congress members have **30-45 days** to disclose trades
|
||||
- 📅 Disclosures are filed as **Periodic Transaction Reports (PTRs)**
|
||||
- 📅 Public databases update **after** filing (usually next day)
|
||||
- 📅 **No real-time feed exists by design**
|
||||
|
||||
**Example Timeline:**
|
||||
```
|
||||
@@ -25,15 +25,15 @@ Feb 17, 2024 → Your system fetches it
|
||||
|
||||
Since trades appear in batches (not continuously), **running once per day is optimal**:
|
||||
|
||||
**Daily (7 AM)** - Catches overnight filings
|
||||
**After market close** - Prices are final
|
||||
**Low server load** - Off-peak hours
|
||||
**Hourly** - Wasteful, no new data
|
||||
**Real-time** - Impossible, not how disclosures work
|
||||
✅ **Daily (7 AM)** - Catches overnight filings
|
||||
✅ **After market close** - Prices are final
|
||||
✅ **Low server load** - Off-peak hours
|
||||
❌ **Hourly** - Wasteful, no new data
|
||||
❌ **Real-time** - Impossible, not how disclosures work
|
||||
|
||||
---
|
||||
|
||||
## Automated Setup Options
|
||||
## 🤖 Automated Setup Options
|
||||
|
||||
### **Option 1: Cron Job (Linux/Proxmox) - Recommended**
|
||||
|
||||
@@ -149,7 +149,7 @@ Or from anywhere:
|
||||
|
||||
---
|
||||
|
||||
## What Gets Updated?
|
||||
## 📊 What Gets Updated?
|
||||
|
||||
### **1. Congressional Trades**
|
||||
**Script:** `fetch_congressional_trades.py`
|
||||
@@ -182,7 +182,7 @@ Or from anywhere:
|
||||
|
||||
---
|
||||
|
||||
## Customizing the Schedule
|
||||
## ⚙️ Customizing the Schedule
|
||||
|
||||
### **Different Frequencies**
|
||||
|
||||
@@ -215,7 +215,7 @@ Or from anywhere:
|
||||
|
||||
---
|
||||
|
||||
## Email Notifications (Optional)
|
||||
## 📧 Email Notifications (Optional)
|
||||
|
||||
### **Setup Email Alerts**
|
||||
|
||||
@@ -310,7 +310,7 @@ python scripts/email_summary.py
|
||||
|
||||
---
|
||||
|
||||
## Monitoring & Logging
|
||||
## 🔍 Monitoring & Logging
|
||||
|
||||
### **Check Cron Job Status**
|
||||
|
||||
@@ -355,7 +355,7 @@ Add to `/etc/logrotate.d/pote`:
|
||||
|
||||
---
|
||||
|
||||
## Handling Failures
|
||||
## 🚨 Handling Failures
|
||||
|
||||
### **What If House Stock Watcher Is Down?**
|
||||
|
||||
@@ -363,7 +363,7 @@ The script is designed to continue even if one step fails:
|
||||
|
||||
```bash
|
||||
# Script continues and logs warnings
|
||||
WARNING: Failed to fetch congressional trades
|
||||
⚠️ WARNING: Failed to fetch congressional trades
|
||||
This is likely because House Stock Watcher API is down
|
||||
Continuing with other steps...
|
||||
```
|
||||
@@ -399,7 +399,7 @@ for attempt in range(MAX_RETRIES):
|
||||
|
||||
---
|
||||
|
||||
## Performance Optimization
|
||||
## 📈 Performance Optimization
|
||||
|
||||
### **Batch Processing**
|
||||
|
||||
@@ -439,7 +439,7 @@ CREATE INDEX IF NOT EXISTS ix_prices_security_id ON prices(security_id);
|
||||
|
||||
---
|
||||
|
||||
## Recommended Setup
|
||||
## 🎯 Recommended Setup
|
||||
|
||||
### **For Proxmox Production:**
|
||||
|
||||
@@ -475,7 +475,7 @@ pote-update
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
## 📝 Summary
|
||||
|
||||
### **Key Points:**
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# Live Market Monitoring + Congressional Trading Analysis
|
||||
|
||||
## What's Possible vs Impossible
|
||||
## 🎯 What's Possible vs Impossible
|
||||
|
||||
### **NOT Possible:**
|
||||
### ❌ **NOT Possible:**
|
||||
- Identify WHO is buying/selling in real-time
|
||||
- Match live trades to specific Congress members
|
||||
- See congressional trades before they're disclosed
|
||||
|
||||
### **IS Possible:**
|
||||
### ✅ **IS Possible:**
|
||||
- Track unusual market activity in real-time
|
||||
- Monitor stocks Congress members historically trade
|
||||
- Compare unusual activity to later disclosures
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
---
|
||||
|
||||
## **Two-Phase Monitoring System**
|
||||
## 🔄 **Two-Phase Monitoring System**
|
||||
|
||||
### **Phase 1: Real-Time Market Monitoring**
|
||||
Monitor unusual activity in stocks Congress trades:
|
||||
@@ -33,7 +33,7 @@ When disclosures come in:
|
||||
|
||||
---
|
||||
|
||||
## **Implementation: Watchlist-Based Monitoring**
|
||||
## 📊 **Implementation: Watchlist-Based Monitoring**
|
||||
|
||||
### **Concept:**
|
||||
|
||||
@@ -57,38 +57,38 @@ Step 3: When Disclosure Appears (30-45 days later)
|
||||
|
||||
---
|
||||
|
||||
## **Data Sources for Live Market Monitoring**
|
||||
## 🛠️ **Data Sources for Live Market Monitoring**
|
||||
|
||||
### **Free/Low-Cost Options:**
|
||||
|
||||
1. **Yahoo Finance (yfinance)**
|
||||
- Real-time quotes (15-min delay free)
|
||||
- Historical options data
|
||||
- Volume data
|
||||
- Not true real-time for options flow
|
||||
- ✅ Real-time quotes (15-min delay free)
|
||||
- ✅ Historical options data
|
||||
- ✅ Volume data
|
||||
- ❌ Not true real-time for options flow
|
||||
|
||||
2. **Unusual Whales API**
|
||||
- Options flow data
|
||||
- Unusual activity alerts
|
||||
- Paid ($50-200/month)
|
||||
- ✅ Options flow data
|
||||
- ✅ Unusual activity alerts
|
||||
- 💰 Paid ($50-200/month)
|
||||
- https://unusualwhales.com/
|
||||
|
||||
3. **Tradier API**
|
||||
- Real-time market data
|
||||
- Options chains
|
||||
- Paid but affordable ($10-50/month)
|
||||
- ✅ Real-time market data
|
||||
- ✅ Options chains
|
||||
- 💰 Paid but affordable ($10-50/month)
|
||||
- https://tradier.com/
|
||||
|
||||
4. **FlowAlgo**
|
||||
- Options flow tracking
|
||||
- Dark pool data
|
||||
- Paid ($99-399/month)
|
||||
- ✅ Options flow tracking
|
||||
- ✅ Dark pool data
|
||||
- 💰 Paid ($99-399/month)
|
||||
- https://www.flowalgo.com/
|
||||
|
||||
5. **Polygon.io**
|
||||
- Real-time stock data
|
||||
- Options data
|
||||
- Free tier + paid plans
|
||||
- ✅ Real-time stock data
|
||||
- ✅ Options data
|
||||
- 💰 Free tier + paid plans
|
||||
- https://polygon.io/
|
||||
|
||||
### **Best Free Option: Build Your Own with yfinance**
|
||||
@@ -97,7 +97,7 @@ Track volume/price changes every 5 minutes for congressional watchlist tickers.
|
||||
|
||||
---
|
||||
|
||||
## **Practical Hybrid System**
|
||||
## 💡 **Practical Hybrid System**
|
||||
|
||||
### **What We Can Build:**
|
||||
|
||||
@@ -141,37 +141,37 @@ for disclosure in new_disclosures:
|
||||
|
||||
---
|
||||
|
||||
## **Example: Nancy Pelosi NVDA Trade Analysis**
|
||||
## 📈 **Example: Nancy Pelosi NVDA Trade Analysis**
|
||||
|
||||
### **Timeline:**
|
||||
|
||||
```
|
||||
Nov 10, 2024:
|
||||
ALERT: NVDA unusual call options activity
|
||||
🔔 ALERT: NVDA unusual call options activity
|
||||
Volume: 10x average
|
||||
Strike: $500 (2 weeks out)
|
||||
|
||||
Nov 15, 2024:
|
||||
Someone buys NVDA (unknown who at the time)
|
||||
💰 Someone buys NVDA (unknown who at the time)
|
||||
|
||||
Nov 18, 2024:
|
||||
NVDA announces new AI chip
|
||||
Stock jumps 15%
|
||||
📰 NVDA announces new AI chip
|
||||
📈 Stock jumps 15%
|
||||
|
||||
Dec 15, 2024:
|
||||
Disclosure: Nancy Pelosi bought NVDA on Nov 15
|
||||
📋 Disclosure: Nancy Pelosi bought NVDA on Nov 15
|
||||
Value: $15,001-$50,000
|
||||
|
||||
ANALYSIS:
|
||||
She bought AFTER unusual options activity (Nov 10)
|
||||
She bought BEFORE announcement (Nov 18)
|
||||
⏱ Timing: 3 days before major news
|
||||
Flag: Investigate if announcement was public knowledge
|
||||
✅ She bought AFTER unusual options activity (Nov 10)
|
||||
❓ She bought BEFORE announcement (Nov 18)
|
||||
⏱️ Timing: 3 days before major news
|
||||
🚩 Flag: Investigate if announcement was public knowledge
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## **Recommended Approach**
|
||||
## 🎯 **Recommended Approach**
|
||||
|
||||
### **Phase 1: Build Congressional Ticker Watchlist**
|
||||
|
||||
@@ -230,12 +230,12 @@ def monitor_tickers(tickers, interval_minutes=5):
|
||||
# Check for unusual volume
|
||||
avg_volume = current['Volume'].mean()
|
||||
if latest['Volume'] > avg_volume * 3:
|
||||
alert(f" {ticker}: Unusual volume spike!")
|
||||
alert(f"🔔 {ticker}: Unusual volume spike!")
|
||||
|
||||
# Check for price movement
|
||||
price_change = (latest['Close'] - current['Open'].iloc[0]) / current['Open'].iloc[0]
|
||||
if abs(price_change) > 0.05: # 5% move
|
||||
alert(f" {ticker}: {price_change:.2%} move today!")
|
||||
alert(f"📈 {ticker}: {price_change:.2%} move today!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error monitoring {ticker}: {e}")
|
||||
@@ -290,31 +290,31 @@ def analyze_disclosure_timing(disclosure):
|
||||
|
||||
---
|
||||
|
||||
## **Realistic Expectations**
|
||||
## 🚨 **Realistic Expectations**
|
||||
|
||||
### **What This System Will Do:**
|
||||
Monitor stocks Congress members historically trade
|
||||
Alert on unusual market activity in those stocks
|
||||
Retroactively correlate disclosures with earlier alerts
|
||||
Identify timing patterns and potential advantages
|
||||
Build database of congressional trading patterns
|
||||
✅ Monitor stocks Congress members historically trade
|
||||
✅ Alert on unusual market activity in those stocks
|
||||
✅ Retroactively correlate disclosures with earlier alerts
|
||||
✅ Identify timing patterns and potential advantages
|
||||
✅ Build database of congressional trading patterns
|
||||
|
||||
### **What This System WON'T Do:**
|
||||
Identify WHO is buying in real-time
|
||||
Give you advance notice of congressional trades
|
||||
Provide real-time inside information
|
||||
Allow you to "front-run" Congress
|
||||
❌ Identify WHO is buying in real-time
|
||||
❌ Give you advance notice of congressional trades
|
||||
❌ Provide real-time inside information
|
||||
❌ Allow you to "front-run" Congress
|
||||
|
||||
### **Legal & Ethical:**
|
||||
All data is public
|
||||
Analysis is retrospective
|
||||
For research and transparency
|
||||
Not market manipulation
|
||||
Cannot and should not be used to replicate potentially illegal trades
|
||||
✅ All data is public
|
||||
✅ Analysis is retrospective
|
||||
✅ For research and transparency
|
||||
✅ Not market manipulation
|
||||
❌ Cannot and should not be used to replicate potentially illegal trades
|
||||
|
||||
---
|
||||
|
||||
## **Proposed Implementation**
|
||||
## 📊 **Proposed Implementation**
|
||||
|
||||
### **New Scripts to Create:**
|
||||
|
||||
@@ -365,15 +365,15 @@ CREATE TABLE disclosure_timing_analysis (
|
||||
|
||||
---
|
||||
|
||||
## **Summary**
|
||||
## 🎯 **Summary**
|
||||
|
||||
### **Your Question:**
|
||||
> "Can we read live trades being made and compare them to a name?"
|
||||
|
||||
### **Answer:**
|
||||
**No** - Live trades are anonymous, can't identify individuals
|
||||
❌ **No** - Live trades are anonymous, can't identify individuals
|
||||
|
||||
**BUT** - You CAN:
|
||||
✅ **BUT** - You CAN:
|
||||
1. Monitor unusual activity in stocks Congress trades
|
||||
2. Log these alerts in real-time
|
||||
3. When disclosures appear (30-45 days later), correlate them
|
||||
@@ -381,25 +381,25 @@ CREATE TABLE disclosure_timing_analysis (
|
||||
5. Build patterns database of timing and performance
|
||||
|
||||
### **This Gives You:**
|
||||
- Transparency on timing advantages
|
||||
- Pattern detection across officials
|
||||
- Research-grade analysis
|
||||
- Historical correlation data
|
||||
- ✅ Transparency on timing advantages
|
||||
- ✅ Pattern detection across officials
|
||||
- ✅ Research-grade analysis
|
||||
- ✅ Historical correlation data
|
||||
|
||||
### **This Does NOT Give You:**
|
||||
- Real-time identity of traders
|
||||
- Advance notice of congressional trades
|
||||
- Ability to "front-run" disclosures
|
||||
- ❌ Real-time identity of traders
|
||||
- ❌ Advance notice of congressional trades
|
||||
- ❌ Ability to "front-run" disclosures
|
||||
|
||||
---
|
||||
|
||||
## **Would You Like Me To Build This?**
|
||||
## 🚀 **Would You Like Me To Build This?**
|
||||
|
||||
I can create:
|
||||
1. Real-time monitoring system for congressional tickers
|
||||
2. Alert logging and analysis
|
||||
3. Timing correlation when disclosures appear
|
||||
4. Pattern detection and reporting
|
||||
1. ✅ Real-time monitoring system for congressional tickers
|
||||
2. ✅ Alert logging and analysis
|
||||
3. ✅ Timing correlation when disclosures appear
|
||||
4. ✅ Pattern detection and reporting
|
||||
|
||||
This would be **Phase 2.5** of POTE - the "timing analysis" module.
|
||||
|
||||
|
||||
@@ -205,18 +205,18 @@ Output:
|
||||
POTE HEALTH CHECK
|
||||
============================================================
|
||||
Timestamp: 2025-12-15T10:30:00
|
||||
Overall Status: OK
|
||||
Overall Status: ✓ OK
|
||||
|
||||
Database Connection: Database connection successful
|
||||
Data Freshness: Data is fresh (2 days old)
|
||||
✓ Database Connection: Database connection successful
|
||||
✓ Data Freshness: Data is fresh (2 days old)
|
||||
latest_trade_date: 2025-12-13
|
||||
Data Counts: Database has 1,234 trades
|
||||
✓ Data Counts: Database has 1,234 trades
|
||||
officials: 45
|
||||
securities: 123
|
||||
trades: 1,234
|
||||
prices: 12,345
|
||||
market_alerts: 567
|
||||
Recent Alerts: 23 alerts in last 24 hours
|
||||
✓ Recent Alerts: 23 alerts in last 24 hours
|
||||
============================================================
|
||||
```
|
||||
|
||||
|
||||
@@ -22,18 +22,18 @@ chmod 600 .env
|
||||
chown poteapp:poteapp .env
|
||||
```
|
||||
|
||||
### Pros
|
||||
### ✅ Pros
|
||||
- Simple, works immediately
|
||||
- No additional setup
|
||||
- Standard practice for Python projects
|
||||
|
||||
### Cons
|
||||
### ⚠️ Cons
|
||||
- Secrets stored in plain text on disk
|
||||
- Risk if server is compromised
|
||||
- No audit trail
|
||||
|
||||
### Security Checklist
|
||||
- [ ] `.env` in `.gitignore` (already done )
|
||||
### 🔒 Security Checklist
|
||||
- [ ] `.env` in `.gitignore` (already done ✅)
|
||||
- [ ] File permissions: `chmod 600 .env`
|
||||
- [ ] Never commit to git
|
||||
- [ ] Backup securely (encrypted)
|
||||
@@ -76,12 +76,12 @@ sudo chmod 600 /etc/systemd/system/pote.service
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
### Pros
|
||||
### ✅ Pros
|
||||
- Secrets not in git or project directory
|
||||
- Standard Linux practice
|
||||
- Works with systemd timers
|
||||
|
||||
### Cons
|
||||
### ⚠️ Cons
|
||||
- Still visible in `systemctl show`
|
||||
- Requires root to edit
|
||||
|
||||
@@ -128,12 +128,12 @@ source .env
|
||||
python scripts/send_daily_report.py
|
||||
```
|
||||
|
||||
### Pros
|
||||
### ✅ Pros
|
||||
- Secrets separate from code
|
||||
- Easy to rotate
|
||||
- Can be backed up separately
|
||||
|
||||
### Cons
|
||||
### ⚠️ Cons
|
||||
- Extra file to manage
|
||||
- Still plain text
|
||||
|
||||
@@ -191,12 +191,12 @@ class Settings(BaseSettings):
|
||||
smtp_password: str = Field(default_factory=lambda: get_secret("SMTP_PASSWORD"))
|
||||
```
|
||||
|
||||
### Pros
|
||||
### ✅ Pros
|
||||
- Docker-native solution
|
||||
- Encrypted in Swarm mode
|
||||
- Never in logs
|
||||
|
||||
### Cons
|
||||
### ⚠️ Cons
|
||||
- Requires Docker
|
||||
- More complex setup
|
||||
|
||||
@@ -228,13 +228,13 @@ secrets = client.secrets.kv.v2.read_secret_version(path='pote')
|
||||
smtp_password = secrets['data']['data']['smtp_password']
|
||||
```
|
||||
|
||||
### Pros
|
||||
### ✅ Pros
|
||||
- Centralized secrets management
|
||||
- Audit logs
|
||||
- Dynamic secrets
|
||||
- Access control
|
||||
|
||||
### Cons
|
||||
### ⚠️ Cons
|
||||
- Complex setup
|
||||
- Requires Vault infrastructure
|
||||
- Overkill for single user
|
||||
@@ -260,14 +260,14 @@ env:
|
||||
DATABASE_URL: postgresql://user:${{ secrets.DB_PASSWORD }}@postgres/db
|
||||
```
|
||||
|
||||
### Important
|
||||
### ⚠️ Important
|
||||
- **Only for CI/CD pipelines**
|
||||
- **NOT for deployed servers**
|
||||
- Secrets are injected during workflow runs
|
||||
|
||||
---
|
||||
|
||||
## Recommendation for Your Setup
|
||||
## 🎯 Recommendation for Your Setup
|
||||
|
||||
### Personal/Research Use (Current)
|
||||
|
||||
@@ -305,9 +305,9 @@ gpg -c .env # Creates .env.gpg
|
||||
|
||||
---
|
||||
|
||||
## General Security Best Practices
|
||||
## 🔒 General Security Best Practices
|
||||
|
||||
### DO
|
||||
### ✅ DO
|
||||
|
||||
- Use strong, unique passwords
|
||||
- Restrict file permissions (`chmod 600`)
|
||||
@@ -316,7 +316,7 @@ gpg -c .env # Creates .env.gpg
|
||||
- Use encrypted backups
|
||||
- Audit who has server access
|
||||
|
||||
### DON'T
|
||||
### ❌ DON'T
|
||||
|
||||
- Commit secrets to git (even private repos)
|
||||
- Store passwords in code
|
||||
@@ -327,7 +327,7 @@ gpg -c .env # Creates .env.gpg
|
||||
|
||||
---
|
||||
|
||||
## Test Your Security
|
||||
## 🧪 Test Your Security
|
||||
|
||||
### Check if `.env` is protected
|
||||
|
||||
@@ -352,7 +352,7 @@ git log --all --full-history --source --pickaxe-all -S 'smtp_password'
|
||||
|
||||
---
|
||||
|
||||
## Password Rotation Procedure
|
||||
## 🔄 Password Rotation Procedure
|
||||
|
||||
### Every 90 days (or if compromised):
|
||||
|
||||
@@ -369,32 +369,32 @@ git log --all --full-history --source --pickaxe-all -S 'smtp_password'
|
||||
|
||||
---
|
||||
|
||||
## Security Level Comparison
|
||||
## 📊 Security Level Comparison
|
||||
|
||||
| Level | Method | Effort | Protection |
|
||||
|-------|--------|--------|------------|
|
||||
| Basic | `.env` (default perms) | None | Low |
|
||||
| Good | `.env` (chmod 600) | 1 min | Medium |
|
||||
| Better | Environment variables | 10 min | Good |
|
||||
| Better | Separate secrets file | 10 min | Good |
|
||||
| Best | Docker Secrets | 30 min | Very Good |
|
||||
| Best | Vault | 2+ hours | Excellent |
|
||||
| 🔓 Basic | `.env` (default perms) | None | Low |
|
||||
| 🔒 Good | `.env` (chmod 600) | 1 min | Medium |
|
||||
| 🔒 Better | Environment variables | 10 min | Good |
|
||||
| 🔒 Better | Separate secrets file | 10 min | Good |
|
||||
| 🔐 Best | Docker Secrets | 30 min | Very Good |
|
||||
| 🔐 Best | Vault | 2+ hours | Excellent |
|
||||
|
||||
---
|
||||
|
||||
## Your Current Status
|
||||
## 🎯 Your Current Status
|
||||
|
||||
**Already secure enough for personal use:**
|
||||
- `.env` in `.gitignore`
|
||||
- Not committed to git
|
||||
- Local server only
|
||||
✅ **Already secure enough for personal use:**
|
||||
- `.env` in `.gitignore` ✅
|
||||
- Not committed to git ✅
|
||||
- Local server only ✅
|
||||
|
||||
**Recommended improvement (2 minutes):**
|
||||
⚠️ **Recommended improvement (2 minutes):**
|
||||
```bash
|
||||
chmod 600 .env
|
||||
```
|
||||
|
||||
**Optional (if paranoid):**
|
||||
🔐 **Optional (if paranoid):**
|
||||
- Use separate secrets file in `/etc/pote/`
|
||||
- Encrypt backups with GPG
|
||||
- Set up password rotation schedule
|
||||
@@ -405,9 +405,9 @@ chmod 600 .env
|
||||
|
||||
**For your levkin.ca setup:**
|
||||
|
||||
1. **Current approach (`.env` file) is fine**
|
||||
1. **Current approach (`.env` file) is fine** ✅
|
||||
2. **Add `chmod 600 .env`** for better security (2 minutes)
|
||||
3. **Don't commit `.env` to git** (already protected )
|
||||
3. **Don't commit `.env` to git** (already protected ✅)
|
||||
4. **Consider upgrading to environment variables** if you deploy to production
|
||||
|
||||
Your current setup is **appropriate for a personal research project**. Don't over-engineer it unless you have specific compliance requirements or a team.
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
# Branch Strategy & Multi-Environment Deployment
|
||||
|
||||
## Overview
|
||||
|
||||
This guide covers setting up a proper Git branching strategy with protected branches for Dev, QA, and Production environments, integrated with your Ansible auto-deployment system.
|
||||
|
||||
---
|
||||
|
||||
## 🌳 Branch Strategy
|
||||
|
||||
### Recommended Branch Structure
|
||||
|
||||
```
|
||||
main (production)
|
||||
├── qa (quality assurance/staging)
|
||||
└── dev (development)
|
||||
```
|
||||
|
||||
**Alternative naming:**
|
||||
```
|
||||
prod (production)
|
||||
├── staging (QA)
|
||||
└── develop (dev)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Branch Protection Rules
|
||||
|
||||
### In Gitea: Repository Settings → Branches
|
||||
|
||||
#### 1. `main` (Production) - MOST PROTECTED
|
||||
|
||||
**Protection Rules:**
|
||||
- ✅ **Require pull request reviews** (at least 1 approval)
|
||||
- ✅ **Require status checks to pass** (CI must pass)
|
||||
- ✅ **Restrict who can push** (only maintainers)
|
||||
- ✅ **Require signed commits** (optional but recommended)
|
||||
- ✅ **Block force pushes**
|
||||
- ✅ **Block deletions**
|
||||
- ✅ **Require linear history** (no merge commits)
|
||||
|
||||
**Merge Strategy:**
|
||||
- Only merge from `qa` branch
|
||||
- Require successful QA testing
|
||||
- Tag releases: `v1.0.0`, `v1.1.0`, etc.
|
||||
|
||||
#### 2. `qa` (Staging) - MODERATELY PROTECTED
|
||||
|
||||
**Protection Rules:**
|
||||
- ✅ **Require pull request reviews** (at least 1 approval)
|
||||
- ✅ **Require status checks to pass** (CI must pass)
|
||||
- ✅ **Block force pushes**
|
||||
- ✅ **Block deletions**
|
||||
|
||||
**Merge Strategy:**
|
||||
- Merge from `dev` branch
|
||||
- Run full test suite
|
||||
- Manual QA testing required
|
||||
|
||||
#### 3. `dev` (Development) - LIGHTLY PROTECTED
|
||||
|
||||
**Protection Rules:**
|
||||
- ✅ **Require status checks to pass** (CI must pass)
|
||||
- ✅ **Block force pushes** (optional)
|
||||
- ⚠️ Allow direct commits (for rapid development)
|
||||
|
||||
**Merge Strategy:**
|
||||
- Feature branches merge here
|
||||
- Continuous integration testing
|
||||
- Auto-deploy to dev environment
|
||||
|
||||
---
|
||||
|
||||
## 📋 Gitea Branch Protection Setup
|
||||
|
||||
### Step-by-Step Configuration
|
||||
|
||||
#### 1. Create Branches
|
||||
|
||||
```bash
|
||||
cd /home/user/Documents/code/pote
|
||||
|
||||
# Create dev branch from main
|
||||
git checkout -b dev
|
||||
git push origin dev
|
||||
|
||||
# Create qa branch from main
|
||||
git checkout -b qa
|
||||
git push origin qa
|
||||
|
||||
# Back to main
|
||||
git checkout main
|
||||
```
|
||||
|
||||
#### 2. Configure in Gitea UI
|
||||
|
||||
```
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
2. Click "Add New Rule"
|
||||
|
||||
For MAIN branch:
|
||||
- Branch name pattern: main
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require 1 approval
|
||||
- ✅ Require status checks
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
- Whitelist: (leave empty or add specific users)
|
||||
|
||||
For QA branch:
|
||||
- Branch name pattern: qa
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require status checks
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
|
||||
For DEV branch:
|
||||
- Branch name pattern: dev
|
||||
- ✅ Require status checks
|
||||
- ⚠️ Allow direct push (for development)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Deployment Workflow Integration
|
||||
|
||||
### Update Workflows for Multi-Environment
|
||||
|
||||
#### 1. Update CI Workflow for All Branches
|
||||
|
||||
**File:** `.github/workflows/ci.yml`
|
||||
|
||||
```yaml
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, qa, dev]
|
||||
pull_request:
|
||||
branches: [main, qa, dev]
|
||||
|
||||
jobs:
|
||||
lint-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
# ... existing CI jobs ...
|
||||
```
|
||||
|
||||
#### 2. Create Environment-Specific Deployment Workflows
|
||||
|
||||
**File:** `.github/workflows/deploy-dev.yml`
|
||||
|
||||
```yaml
|
||||
name: Deploy to Dev
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy-dev:
|
||||
runs-on: ubuntu-latest
|
||||
environment: development
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Deploy to Dev Server
|
||||
env:
|
||||
DEV_HOST: ${{ secrets.DEV_HOST }}
|
||||
DEV_USER: ${{ secrets.DEV_USER }}
|
||||
DEV_SSH_KEY: ${{ secrets.DEV_SSH_KEY }}
|
||||
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_DEV }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_DEV }}
|
||||
run: |
|
||||
# Setup SSH
|
||||
mkdir -p ~/.ssh
|
||||
echo "$DEV_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -H $DEV_HOST >> ~/.ssh/known_hosts
|
||||
|
||||
# Deploy
|
||||
ssh ${DEV_USER}@${DEV_HOST} << 'ENDSSH'
|
||||
cd ~/pote-dev
|
||||
git fetch origin
|
||||
git checkout dev
|
||||
git pull origin dev
|
||||
source venv/bin/activate
|
||||
pip install -e .
|
||||
alembic upgrade head
|
||||
ENDSSH
|
||||
```
|
||||
|
||||
**File:** `.github/workflows/deploy-qa.yml`
|
||||
|
||||
```yaml
|
||||
name: Deploy to QA
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [qa]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy-qa:
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Deploy to QA Server
|
||||
env:
|
||||
QA_HOST: ${{ secrets.QA_HOST }}
|
||||
QA_USER: ${{ secrets.QA_USER }}
|
||||
QA_SSH_KEY: ${{ secrets.QA_SSH_KEY }}
|
||||
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_QA }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_QA }}
|
||||
run: |
|
||||
# Setup SSH
|
||||
mkdir -p ~/.ssh
|
||||
echo "$QA_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -H $QA_HOST >> ~/.ssh/known_hosts
|
||||
|
||||
# Deploy
|
||||
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
|
||||
cd ~/pote-qa
|
||||
git fetch origin
|
||||
git checkout qa
|
||||
git pull origin qa
|
||||
source venv/bin/activate
|
||||
pip install -e .
|
||||
alembic upgrade head
|
||||
ENDSSH
|
||||
|
||||
- name: Run Smoke Tests
|
||||
run: |
|
||||
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
|
||||
cd ~/pote-qa
|
||||
source venv/bin/activate
|
||||
python scripts/health_check.py
|
||||
ENDSSH
|
||||
```
|
||||
|
||||
**File:** `.github/workflows/deploy-prod.yml`
|
||||
|
||||
```yaml
|
||||
name: Deploy to Production
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirm:
|
||||
description: 'Type "DEPLOY" to confirm production deployment'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
deploy-prod:
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
if: github.event.inputs.confirm == 'DEPLOY' || github.event_name == 'push'
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Create Release Tag
|
||||
run: |
|
||||
git tag -a "v$(date +%Y%m%d-%H%M%S)" -m "Production release"
|
||||
git push origin --tags
|
||||
|
||||
- name: Deploy to Production
|
||||
env:
|
||||
PROD_HOST: ${{ secrets.PROXMOX_HOST }}
|
||||
PROD_USER: ${{ secrets.PROXMOX_USER }}
|
||||
PROD_SSH_KEY: ${{ secrets.PROXMOX_SSH_KEY }}
|
||||
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
run: |
|
||||
# Setup SSH
|
||||
mkdir -p ~/.ssh
|
||||
echo "$PROD_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -H $PROD_HOST >> ~/.ssh/known_hosts
|
||||
|
||||
# Backup current production
|
||||
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
|
||||
cd ~/pote
|
||||
git tag "backup-$(date +%Y%m%d-%H%M%S)"
|
||||
ENDSSH
|
||||
|
||||
# Deploy
|
||||
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
|
||||
cd ~/pote
|
||||
git fetch origin
|
||||
git checkout main
|
||||
git pull origin main
|
||||
source venv/bin/activate
|
||||
pip install -e .
|
||||
alembic upgrade head
|
||||
ENDSSH
|
||||
|
||||
- name: Health Check
|
||||
run: |
|
||||
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
|
||||
cd ~/pote
|
||||
source venv/bin/activate
|
||||
python scripts/health_check.py
|
||||
ENDSSH
|
||||
|
||||
- name: Rollback on Failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "❌ Deployment failed, rolling back..."
|
||||
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
|
||||
cd ~/pote
|
||||
latest_backup=$(git tag -l "backup-*" | sort -r | head -1)
|
||||
git checkout "$latest_backup"
|
||||
source venv/bin/activate
|
||||
alembic upgrade head
|
||||
ENDSSH
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Gitea Secrets for Multi-Environment
|
||||
|
||||
### Organize Secrets by Environment
|
||||
|
||||
#### Development Secrets
|
||||
```
|
||||
DEV_HOST=10.0.10.100
|
||||
DEV_USER=poteapp
|
||||
DEV_SSH_KEY=(dev SSH key)
|
||||
SMTP_PASSWORD_DEV=(dev mail password)
|
||||
DB_PASSWORD_DEV=dev_password_123
|
||||
```
|
||||
|
||||
#### QA/Staging Secrets
|
||||
```
|
||||
QA_HOST=10.0.10.101
|
||||
QA_USER=poteapp
|
||||
QA_SSH_KEY=(qa SSH key)
|
||||
SMTP_PASSWORD_QA=(qa mail password)
|
||||
DB_PASSWORD_QA=qa_password_123
|
||||
```
|
||||
|
||||
#### Production Secrets
|
||||
```
|
||||
PROXMOX_HOST=10.0.10.95
|
||||
PROXMOX_USER=poteapp
|
||||
PROXMOX_SSH_KEY=(prod SSH key)
|
||||
SMTP_PASSWORD=(prod mail password)
|
||||
DB_PASSWORD=changeme123
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Deployment Flow Diagram
|
||||
|
||||
```
|
||||
Developer
|
||||
│
|
||||
├─> Commit to feature branch
|
||||
│
|
||||
├─> Create PR to dev
|
||||
│ │
|
||||
│ ├─> CI runs (tests)
|
||||
│ │
|
||||
│ ├─> Merge to dev
|
||||
│ │
|
||||
│ └─> Auto-deploy to DEV environment
|
||||
│
|
||||
├─> Test in DEV
|
||||
│
|
||||
├─> Create PR: dev → qa
|
||||
│ │
|
||||
│ ├─> CI runs (tests)
|
||||
│ │
|
||||
│ ├─> Code review required
|
||||
│ │
|
||||
│ ├─> Merge to qa
|
||||
│ │
|
||||
│ └─> Auto-deploy to QA environment
|
||||
│
|
||||
├─> QA Testing
|
||||
│
|
||||
└─> Create PR: qa → main
|
||||
│
|
||||
├─> CI runs (tests)
|
||||
│
|
||||
├─> Code review required (2 approvals)
|
||||
│
|
||||
├─> Manual approval for prod deploy
|
||||
│
|
||||
├─> Merge to main
|
||||
│
|
||||
├─> Create release tag
|
||||
│
|
||||
└─> Deploy to PRODUCTION
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Integration with Your Ansible System
|
||||
|
||||
### Option 1: Gitea Webhooks → Ansible
|
||||
|
||||
**Setup:**
|
||||
|
||||
1. **In Gitea:** Settings → Webhooks → Add Webhook
|
||||
- URL: `https://your-ansible-controller/webhook/pote`
|
||||
- Trigger: Push events
|
||||
- Branches: `dev`, `qa`, `main`
|
||||
|
||||
2. **Ansible Playbook:** `deploy-pote.yml`
|
||||
|
||||
```yaml
|
||||
---
|
||||
- name: Deploy POTE based on branch
|
||||
hosts: "{{ target_env }}"
|
||||
vars:
|
||||
branch: "{{ git_branch }}"
|
||||
env: "{{ target_env }}"
|
||||
tasks:
|
||||
- name: Pull latest code
|
||||
git:
|
||||
repo: gitea@10.0.30.169:ilia/POTE.git
|
||||
dest: /home/poteapp/pote
|
||||
version: "{{ branch }}"
|
||||
force: yes
|
||||
|
||||
- name: Install dependencies
|
||||
pip:
|
||||
requirements: /home/poteapp/pote/requirements.txt
|
||||
virtualenv: /home/poteapp/pote/venv
|
||||
|
||||
- name: Run migrations
|
||||
command: alembic upgrade head
|
||||
args:
|
||||
chdir: /home/poteapp/pote
|
||||
environment:
|
||||
DATABASE_URL: "{{ database_url }}"
|
||||
|
||||
- name: Update secrets
|
||||
template:
|
||||
src: env.j2
|
||||
dest: /home/poteapp/pote/.env
|
||||
mode: 0600
|
||||
|
||||
- name: Health check
|
||||
command: python scripts/health_check.py
|
||||
args:
|
||||
chdir: /home/poteapp/pote
|
||||
```
|
||||
|
||||
3. **Ansible Inventory:** `inventory.yml`
|
||||
|
||||
```yaml
|
||||
all:
|
||||
children:
|
||||
development:
|
||||
hosts:
|
||||
dev-pote:
|
||||
ansible_host: 10.0.10.100
|
||||
target_env: development
|
||||
git_branch: dev
|
||||
database_url: postgresql://poteuser:dev_pass@localhost/potedb_dev
|
||||
|
||||
staging:
|
||||
hosts:
|
||||
qa-pote:
|
||||
ansible_host: 10.0.10.101
|
||||
target_env: staging
|
||||
git_branch: qa
|
||||
database_url: postgresql://poteuser:qa_pass@localhost/potedb_qa
|
||||
|
||||
production:
|
||||
hosts:
|
||||
prod-pote:
|
||||
ansible_host: 10.0.10.95
|
||||
target_env: production
|
||||
git_branch: main
|
||||
database_url: postgresql://poteuser:prod_pass@localhost/potedb
|
||||
```
|
||||
|
||||
### Option 2: Gitea Actions → Ansible
|
||||
|
||||
**File:** `.github/workflows/ansible-deploy.yml`
|
||||
|
||||
```yaml
|
||||
name: Ansible Deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, qa, dev]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Determine environment
|
||||
id: env
|
||||
run: |
|
||||
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
|
||||
echo "environment=production" >> $GITHUB_OUTPUT
|
||||
echo "host=10.0.10.95" >> $GITHUB_OUTPUT
|
||||
elif [ "${{ github.ref }}" == "refs/heads/qa" ]; then
|
||||
echo "environment=staging" >> $GITHUB_OUTPUT
|
||||
echo "host=10.0.10.101" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "environment=development" >> $GITHUB_OUTPUT
|
||||
echo "host=10.0.10.100" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Trigger Ansible
|
||||
run: |
|
||||
curl -X POST https://your-ansible-controller/api/deploy \
|
||||
-H "Authorization: Bearer ${{ secrets.ANSIBLE_TOKEN }}" \
|
||||
-d '{
|
||||
"project": "pote",
|
||||
"environment": "${{ steps.env.outputs.environment }}",
|
||||
"branch": "${{ github.ref_name }}",
|
||||
"commit": "${{ github.sha }}"
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ✅ Complete Setup Checklist
|
||||
|
||||
### 1. Git Configuration
|
||||
- [ ] Create `dev` branch
|
||||
- [ ] Create `qa` branch
|
||||
- [ ] Keep `main` branch
|
||||
- [ ] Push all branches to Gitea
|
||||
|
||||
### 2. Gitea Branch Protection
|
||||
- [ ] Protect `main` (require PR + approval + CI)
|
||||
- [ ] Protect `qa` (require PR + CI)
|
||||
- [ ] Configure `dev` (require CI only)
|
||||
|
||||
### 3. Gitea Secrets
|
||||
- [ ] Add DEV environment secrets
|
||||
- [ ] Add QA environment secrets
|
||||
- [ ] Add PROD environment secrets
|
||||
|
||||
### 4. Workflows
|
||||
- [ ] Update `ci.yml` for all branches
|
||||
- [ ] Create `deploy-dev.yml`
|
||||
- [ ] Create `deploy-qa.yml`
|
||||
- [ ] Create `deploy-prod.yml`
|
||||
|
||||
### 5. Ansible Integration
|
||||
- [ ] Configure Gitea webhooks (if using webhooks)
|
||||
- [ ] Update Ansible playbooks for POTE
|
||||
- [ ] Test deployment to each environment
|
||||
|
||||
### 6. Documentation
|
||||
- [ ] Document deployment process
|
||||
- [ ] Create runbook for rollbacks
|
||||
- [ ] Train team on workflow
|
||||
|
||||
---
|
||||
|
||||
## 🚨 What You're Missing (Important!)
|
||||
|
||||
### 1. **Environment Variables per Environment**
|
||||
|
||||
Create separate `.env` files:
|
||||
- `.env.dev`
|
||||
- `.env.qa`
|
||||
- `.env.prod`
|
||||
|
||||
**Never commit these!** Use Ansible templates or Gitea secrets.
|
||||
|
||||
### 2. **Database Migrations Strategy**
|
||||
|
||||
```bash
|
||||
# Test migrations in dev first
|
||||
alembic upgrade head # dev
|
||||
|
||||
# Then qa
|
||||
alembic upgrade head # qa
|
||||
|
||||
# Finally prod (with backup!)
|
||||
pg_dump potedb > backup.sql
|
||||
alembic upgrade head # prod
|
||||
```
|
||||
|
||||
### 3. **Rollback Strategy**
|
||||
|
||||
```bash
|
||||
# Git rollback
|
||||
git checkout <previous-commit>
|
||||
|
||||
# Database rollback
|
||||
alembic downgrade -1
|
||||
|
||||
# Or restore from backup
|
||||
psql potedb < backup.sql
|
||||
```
|
||||
|
||||
### 4. **Monitoring & Alerts**
|
||||
|
||||
- Health checks after each deployment
|
||||
- Email/Slack notifications on failure
|
||||
- Automated rollback on critical errors
|
||||
|
||||
### 5. **Feature Flags**
|
||||
|
||||
Consider adding feature flags for gradual rollouts:
|
||||
|
||||
```python
|
||||
# In config.py
|
||||
FEATURE_NEW_ANALYTICS = os.getenv("FEATURE_NEW_ANALYTICS", "false") == "true"
|
||||
```
|
||||
|
||||
### 6. **Changelog & Release Notes**
|
||||
|
||||
Maintain `CHANGELOG.md`:
|
||||
|
||||
```markdown
|
||||
## [1.2.0] - 2025-12-15
|
||||
### Added
|
||||
- Email reporting system
|
||||
- Gitea secrets integration
|
||||
|
||||
### Fixed
|
||||
- Database connection timeout
|
||||
|
||||
### Changed
|
||||
- Improved error handling
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 Quick Reference Commands
|
||||
|
||||
```bash
|
||||
# Create branches
|
||||
git checkout -b dev && git push origin dev
|
||||
git checkout -b qa && git push origin qa
|
||||
|
||||
# Merge dev → qa
|
||||
git checkout qa
|
||||
git merge dev
|
||||
git push origin qa
|
||||
|
||||
# Merge qa → main (via PR in Gitea!)
|
||||
# Don't do this directly - use Pull Request
|
||||
|
||||
# Check which branch you're on
|
||||
git branch
|
||||
|
||||
# See all branches
|
||||
git branch -a
|
||||
|
||||
# Deploy manually (if Ansible fails)
|
||||
ssh poteapp@10.0.10.100 "cd ~/pote && git pull origin dev"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Recommended Next Steps
|
||||
|
||||
1. **Right now:** Create dev and qa branches
|
||||
2. **Today:** Set up branch protection in Gitea
|
||||
3. **This week:** Create environment-specific workflows
|
||||
4. **This week:** Integrate with your Ansible system
|
||||
5. **Next week:** Test full deployment flow
|
||||
|
||||
---
|
||||
|
||||
**With this setup, you'll have a professional, production-ready deployment pipeline!** 🚀
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
# ✅ Branch Strategy Setup Complete!
|
||||
|
||||
## 🌳 Branches Created
|
||||
|
||||
Your POTE repository now has three branches:
|
||||
|
||||
```
|
||||
✅ main (production) - PROTECTED
|
||||
✅ qa (staging) - Ready to protect
|
||||
✅ dev (development) - Ready to protect
|
||||
```
|
||||
|
||||
**Current status:**
|
||||
- `main` is already protected (you saw the error - that's good!)
|
||||
- New documentation committed to `dev` branch
|
||||
- Ready to configure protection for `qa` and `dev`
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Next Steps: Configure Branch Protection
|
||||
|
||||
### Go to Gitea: https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
|
||||
### 1. Protect `main` (Production) - Already Done! ✅
|
||||
|
||||
Your `main` branch is already protected (we couldn't push directly to it).
|
||||
|
||||
**Verify settings:**
|
||||
- Branch name pattern: `main`
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require approvals: 1 (or 2 for production)
|
||||
- ✅ Require status checks
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
|
||||
### 2. Protect `qa` (Staging) - TODO
|
||||
|
||||
Click "Add New Rule":
|
||||
- Branch name pattern: `qa`
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require 1 approval
|
||||
- ✅ Require status checks to pass
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
|
||||
### 3. Configure `dev` (Development) - TODO
|
||||
|
||||
Click "Add New Rule":
|
||||
- Branch name pattern: `dev`
|
||||
- ✅ Require status checks to pass (CI must pass)
|
||||
- ⚠️ Allow direct push (for rapid development)
|
||||
- ✅ Block force push (optional)
|
||||
|
||||
---
|
||||
|
||||
## 📋 What You're Missing (Checklist)
|
||||
|
||||
### ✅ Already Have:
|
||||
- [x] Three branches (main, qa, dev)
|
||||
- [x] Main branch protection
|
||||
- [x] Comprehensive documentation
|
||||
- [x] CI/CD pipeline
|
||||
- [x] Gitea secrets integration
|
||||
|
||||
### 🔲 Need to Add:
|
||||
|
||||
#### 1. **Environment-Specific Secrets in Gitea**
|
||||
|
||||
Go to: https://git.levkin.ca/ilia/POTE/settings/secrets
|
||||
|
||||
**Development:**
|
||||
```
|
||||
DEV_HOST=10.0.10.100 (or your dev server IP)
|
||||
DEV_USER=poteapp
|
||||
DEV_SSH_KEY=(SSH key for dev server)
|
||||
SMTP_PASSWORD_DEV=(dev email password)
|
||||
DB_PASSWORD_DEV=dev_password_123
|
||||
```
|
||||
|
||||
**QA/Staging:**
|
||||
```
|
||||
QA_HOST=10.0.10.101 (or your QA server IP)
|
||||
QA_USER=poteapp
|
||||
QA_SSH_KEY=(SSH key for QA server)
|
||||
SMTP_PASSWORD_QA=(qa email password)
|
||||
DB_PASSWORD_QA=qa_password_123
|
||||
```
|
||||
|
||||
**Production:**
|
||||
```
|
||||
PROXMOX_HOST=10.0.10.95 (already have this)
|
||||
PROXMOX_USER=poteapp
|
||||
PROXMOX_SSH_KEY=(already have this)
|
||||
SMTP_PASSWORD=(already have this)
|
||||
DB_PASSWORD=changeme123
|
||||
```
|
||||
|
||||
#### 2. **Create Environment-Specific Deployment Workflows**
|
||||
|
||||
Files to create:
|
||||
- `.github/workflows/deploy-dev.yml` (see docs/14_branch_strategy_and_deployment.md)
|
||||
- `.github/workflows/deploy-qa.yml`
|
||||
- `.github/workflows/deploy-prod.yml` (already have deploy.yml, can rename/update)
|
||||
|
||||
#### 3. **Set Up Separate Servers/Containers**
|
||||
|
||||
You need three environments:
|
||||
|
||||
| Environment | Server/Container | Database | Purpose |
|
||||
|-------------|------------------|----------|---------|
|
||||
| **Dev** | `10.0.10.100` (or new LXC) | `potedb_dev` | Development testing |
|
||||
| **QA** | `10.0.10.101` (or new LXC) | `potedb_qa` | Pre-production testing |
|
||||
| **Prod** | `10.0.10.95` (existing) | `potedb` | Production |
|
||||
|
||||
**Options:**
|
||||
- Create 2 more LXC containers (recommended)
|
||||
- Use same server with different ports/databases
|
||||
- Use Docker containers
|
||||
|
||||
#### 4. **Ansible Integration**
|
||||
|
||||
**Option A: Gitea Webhooks**
|
||||
```
|
||||
Gitea → Settings → Webhooks → Add Webhook
|
||||
URL: https://your-ansible-controller/webhook/pote
|
||||
Trigger on: Push events
|
||||
Branches: dev, qa, main
|
||||
```
|
||||
|
||||
**Option B: Gitea Actions calls Ansible**
|
||||
```yaml
|
||||
# In workflow
|
||||
- name: Trigger Ansible
|
||||
run: |
|
||||
curl -X POST https://ansible-controller/api/deploy \
|
||||
-d '{"branch": "${{ github.ref_name }}"}'
|
||||
```
|
||||
|
||||
#### 5. **Update Ansible Playbook**
|
||||
|
||||
Your Ansible playbook should:
|
||||
```yaml
|
||||
- name: Deploy POTE
|
||||
hosts: "{{ target_env }}"
|
||||
vars:
|
||||
branch: "{{ git_branch }}" # dev, qa, or main
|
||||
tasks:
|
||||
- git:
|
||||
repo: gitea@10.0.30.169:ilia/POTE.git
|
||||
dest: /home/poteapp/pote
|
||||
version: "{{ branch }}"
|
||||
# ... rest of deployment
|
||||
```
|
||||
|
||||
#### 6. **Database Migration Strategy**
|
||||
|
||||
```bash
|
||||
# Always test in dev first
|
||||
ssh poteapp@dev-server "cd ~/pote && alembic upgrade head"
|
||||
|
||||
# Then QA
|
||||
ssh poteapp@qa-server "cd ~/pote && alembic upgrade head"
|
||||
|
||||
# Finally prod (with backup!)
|
||||
ssh poteapp@prod-server "pg_dump potedb > backup.sql && cd ~/pote && alembic upgrade head"
|
||||
```
|
||||
|
||||
#### 7. **Monitoring & Alerts**
|
||||
|
||||
Add to each deployment:
|
||||
```yaml
|
||||
- name: Health Check
|
||||
run: python scripts/health_check.py
|
||||
|
||||
- name: Send Alert on Failure
|
||||
if: failure()
|
||||
run: |
|
||||
# Send email/Slack notification
|
||||
```
|
||||
|
||||
#### 8. **Environment Variables**
|
||||
|
||||
Create separate configs:
|
||||
- `.env.dev` (in dev server)
|
||||
- `.env.qa` (in qa server)
|
||||
- `.env` (in prod server - already have)
|
||||
|
||||
**Never commit these!** Use Ansible templates or deployment workflows.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Workflow After Setup
|
||||
|
||||
### Development Flow:
|
||||
|
||||
```bash
|
||||
# 1. Work on feature
|
||||
git checkout dev
|
||||
git pull origin dev
|
||||
# ... make changes ...
|
||||
git commit -m "Add feature"
|
||||
git push origin dev
|
||||
|
||||
# 2. Auto-deploys to DEV server
|
||||
# (via Gitea webhook or Actions)
|
||||
|
||||
# 3. Test in DEV
|
||||
|
||||
# 4. Promote to QA
|
||||
# Create PR: dev → qa in Gitea UI
|
||||
# Merge after approval
|
||||
# Auto-deploys to QA server
|
||||
|
||||
# 5. QA Testing
|
||||
|
||||
# 6. Promote to PROD
|
||||
# Create PR: qa → main in Gitea UI
|
||||
# Requires 2 approvals
|
||||
# Merge
|
||||
# Manual deployment trigger (with confirmation)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
**Main guide:** `docs/14_branch_strategy_and_deployment.md`
|
||||
|
||||
Covers:
|
||||
- ✅ Branch protection setup
|
||||
- ✅ Multi-environment workflows
|
||||
- ✅ Ansible integration
|
||||
- ✅ Deployment flow
|
||||
- ✅ Rollback procedures
|
||||
- ✅ Complete checklist
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Quick Actions (Do These Now)
|
||||
|
||||
### 1. Configure Branch Protection (5 minutes)
|
||||
|
||||
```
|
||||
https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
- Add rule for 'qa'
|
||||
- Add rule for 'dev'
|
||||
```
|
||||
|
||||
### 2. Add Environment Secrets (10 minutes)
|
||||
|
||||
```
|
||||
https://git.levkin.ca/ilia/POTE/settings/secrets
|
||||
- Add DEV_* secrets
|
||||
- Add QA_* secrets
|
||||
- Verify PROD secrets exist
|
||||
```
|
||||
|
||||
### 3. Create PR for Documentation (2 minutes)
|
||||
|
||||
```
|
||||
https://git.levkin.ca/ilia/POTE/compare/main...dev
|
||||
- Create pull request
|
||||
- Title: "Add branch strategy documentation"
|
||||
- Merge to main
|
||||
```
|
||||
|
||||
### 4. Decide on Server Setup
|
||||
|
||||
**Option 1:** Create 2 more LXC containers
|
||||
```bash
|
||||
# On Proxmox host
|
||||
pct clone 100 101 --hostname pote-dev
|
||||
pct clone 100 102 --hostname pote-qa
|
||||
```
|
||||
|
||||
**Option 2:** Use existing server with different databases
|
||||
```bash
|
||||
# On existing server
|
||||
createdb potedb_dev
|
||||
createdb potedb_qa
|
||||
```
|
||||
|
||||
### 5. Configure Ansible
|
||||
|
||||
Update your Ansible inventory to include:
|
||||
- `pote-dev` host
|
||||
- `pote-qa` host
|
||||
- `pote-prod` host (existing)
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Important Notes
|
||||
|
||||
### Main Branch is Protected!
|
||||
|
||||
You saw this error:
|
||||
```
|
||||
remote: Gitea: Not allowed to push to protected branch main
|
||||
```
|
||||
|
||||
**This is GOOD!** It means:
|
||||
- ✅ Main branch is protected
|
||||
- ✅ Can't accidentally push directly
|
||||
- ✅ Must use Pull Requests
|
||||
- ✅ Requires code review
|
||||
|
||||
**To update main:**
|
||||
1. Push to `dev` or `qa`
|
||||
2. Create Pull Request in Gitea
|
||||
3. Get approval
|
||||
4. Merge
|
||||
|
||||
### Current Branch Status
|
||||
|
||||
```bash
|
||||
$ git branch
|
||||
dev ← New documentation is here
|
||||
* main ← Protected, can't push directly
|
||||
qa ← Empty, same as main
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
- **Repository:** https://git.levkin.ca/ilia/POTE
|
||||
- **Branch Protection:** https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
- **Secrets:** https://git.levkin.ca/ilia/POTE/settings/secrets
|
||||
- **Actions:** https://git.levkin.ca/ilia/POTE/actions
|
||||
- **Create PR:** https://git.levkin.ca/ilia/POTE/compare/main...dev
|
||||
|
||||
---
|
||||
|
||||
## ✅ Summary
|
||||
|
||||
**What's Done:**
|
||||
- ✅ Created `dev`, `qa`, `main` branches
|
||||
- ✅ Main branch is protected
|
||||
- ✅ Documentation committed to `dev`
|
||||
- ✅ Ready for Ansible integration
|
||||
|
||||
**What's Next:**
|
||||
1. Configure branch protection for `qa` and `dev`
|
||||
2. Add environment-specific secrets
|
||||
3. Create PR to merge docs to main
|
||||
4. Set up dev/qa servers
|
||||
5. Configure Ansible for multi-environment
|
||||
6. Test deployment flow
|
||||
|
||||
**You're 80% there! Just need to configure Gitea settings and set up the additional servers.** 🚀
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
# 🔧 Pipeline Setup Guide for Branch Protection
|
||||
|
||||
## ❓ Do You Need a Pipeline?
|
||||
|
||||
**YES!** If you want to use "Require status checks" in branch protection, you need a CI pipeline.
|
||||
|
||||
**Good news:** You already have one! ✅
|
||||
|
||||
---
|
||||
|
||||
## ✅ What You Already Have
|
||||
|
||||
### CI Pipeline: `.github/workflows/ci.yml`
|
||||
|
||||
**Status:** ✅ Exists and working
|
||||
**Runs on:** Push to `main`, `qa`, `dev` (just updated!)
|
||||
**What it does:**
|
||||
- Runs linters (ruff, black, mypy)
|
||||
- Runs 93 tests
|
||||
- Checks code quality
|
||||
- Uses PostgreSQL for integration tests
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Setup Order (IMPORTANT!)
|
||||
|
||||
### ⚠️ **DO THIS IN ORDER:**
|
||||
|
||||
### Step 1: Merge CI Updates to Main (FIRST!)
|
||||
|
||||
**Why:** Branch protection needs the CI pipeline to exist in the branch you're protecting.
|
||||
|
||||
**How:**
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
|
||||
2. Click "New Pull Request"
|
||||
3. Title: "Update CI for multi-branch support"
|
||||
4. **Merge this PR** (you can merge without protection for now)
|
||||
|
||||
**What this does:**
|
||||
- Updates CI to run on `main`, `qa`, and `dev`
|
||||
- Makes CI available for branch protection
|
||||
|
||||
---
|
||||
|
||||
### Step 2: Verify CI is Working
|
||||
|
||||
After merging the PR, check:
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/actions
|
||||
2. You should see the CI workflow running
|
||||
3. Wait for it to complete (green checkmark ✅)
|
||||
|
||||
**If CI fails:**
|
||||
- Don't set up branch protection yet
|
||||
- Fix the CI issues first
|
||||
- Ensure tests pass
|
||||
|
||||
---
|
||||
|
||||
### Step 3: Configure Branch Protection (AFTER CI WORKS)
|
||||
|
||||
**Only after CI is passing**, go to:
|
||||
https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
|
||||
#### For `main` Branch (Already Protected)
|
||||
|
||||
**Verify these settings:**
|
||||
- Branch pattern: `main`
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require 1-2 approvals
|
||||
- ✅ **Require status checks to pass before merging**
|
||||
- Select: `CI / lint-and-test` (this appears after CI runs once)
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
|
||||
#### For `qa` Branch
|
||||
|
||||
Click "Add New Rule":
|
||||
- Branch pattern: `qa`
|
||||
- ✅ Enable push protection
|
||||
- ✅ Require pull request
|
||||
- ✅ Require 1 approval
|
||||
- ✅ **Require status checks to pass before merging**
|
||||
- Select: `CI / lint-and-test`
|
||||
- ✅ Block force push
|
||||
- ✅ Block deletion
|
||||
|
||||
#### For `dev` Branch
|
||||
|
||||
Click "Add New Rule":
|
||||
- Branch pattern: `dev`
|
||||
- ✅ **Require status checks to pass before merging**
|
||||
- Select: `CI / lint-and-test`
|
||||
- ⚠️ **Allow direct push** (no PR required for dev)
|
||||
- ⚠️ Allow force push (optional, for rebasing)
|
||||
|
||||
---
|
||||
|
||||
## 🔍 What "Require Status Checks" Means
|
||||
|
||||
When you enable "Require status checks":
|
||||
|
||||
**Before merge:**
|
||||
```
|
||||
PR created: dev → qa
|
||||
↓
|
||||
CI pipeline runs automatically
|
||||
↓
|
||||
Tests must pass ✅
|
||||
↓
|
||||
Only then can you merge
|
||||
```
|
||||
|
||||
**If CI fails:**
|
||||
```
|
||||
PR created: dev → qa
|
||||
↓
|
||||
CI pipeline runs
|
||||
↓
|
||||
Tests fail ❌
|
||||
↓
|
||||
Merge button is DISABLED
|
||||
↓
|
||||
Must fix code and push again
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📋 Status Checks Available
|
||||
|
||||
After your CI runs once, you'll see these options in branch protection:
|
||||
|
||||
**Available checks:**
|
||||
- `CI / lint-and-test` - Main CI pipeline (93 tests)
|
||||
- `CI / security-scan` - Security scanning
|
||||
- `CI / dependency-scan` - Dependency vulnerabilities
|
||||
- `CI / docker-build-test` - Docker build verification
|
||||
|
||||
**Recommended:**
|
||||
- **For `main`:** Require ALL checks ✅
|
||||
- **For `qa`:** Require `lint-and-test` + `security-scan` ✅
|
||||
- **For `dev`:** Require `lint-and-test` only ✅
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Step-by-Step Setup (Complete)
|
||||
|
||||
### 1. Merge CI Updates (5 minutes)
|
||||
|
||||
```
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
|
||||
2. Create PR: "Update CI for multi-branch support"
|
||||
3. Merge (you can approve your own PR for now)
|
||||
4. Wait for CI to run on main branch
|
||||
```
|
||||
|
||||
### 2. Check CI Status (2 minutes)
|
||||
|
||||
```
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/actions
|
||||
2. Click on the latest workflow run
|
||||
3. Verify all jobs pass ✅
|
||||
```
|
||||
|
||||
### 3. Configure Branch Protection (10 minutes)
|
||||
|
||||
```
|
||||
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
|
||||
2. For main (update existing rule):
|
||||
- ✅ Require status checks
|
||||
- Select: CI / lint-and-test
|
||||
|
||||
3. Add rule for qa:
|
||||
- Branch: qa
|
||||
- ✅ Require PR
|
||||
- ✅ Require status checks
|
||||
- Select: CI / lint-and-test
|
||||
|
||||
4. Add rule for dev:
|
||||
- Branch: dev
|
||||
- ✅ Require status checks
|
||||
- Select: CI / lint-and-test
|
||||
- ⚠️ Allow direct push
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Common Issues
|
||||
|
||||
### Issue 1: "No status checks found"
|
||||
|
||||
**Cause:** CI hasn't run on that branch yet
|
||||
|
||||
**Fix:**
|
||||
```bash
|
||||
# Push something to trigger CI
|
||||
git checkout dev
|
||||
git commit --allow-empty -m "Trigger CI"
|
||||
git push origin dev
|
||||
|
||||
# Wait for CI to run, then configure protection
|
||||
```
|
||||
|
||||
### Issue 2: "Status check never completes"
|
||||
|
||||
**Cause:** CI is failing or stuck
|
||||
|
||||
**Fix:**
|
||||
1. Go to Actions tab
|
||||
2. Check the failing job
|
||||
3. Fix the issue
|
||||
4. Push again
|
||||
|
||||
### Issue 3: "Can't select status checks in dropdown"
|
||||
|
||||
**Cause:** CI workflow name doesn't match
|
||||
|
||||
**Fix:**
|
||||
- Workflow must be named exactly: `CI`
|
||||
- Job must be named: `lint-and-test`
|
||||
- Already correct in your `.github/workflows/ci.yml` ✅
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Test Your Setup
|
||||
|
||||
### After configuring protection:
|
||||
|
||||
**Test 1: Try to push directly to main**
|
||||
```bash
|
||||
git checkout main
|
||||
git commit --allow-empty -m "Test"
|
||||
git push origin main
|
||||
# Should fail: "Not allowed to push to protected branch"
|
||||
```
|
||||
|
||||
**Test 2: Create PR with failing tests**
|
||||
```bash
|
||||
git checkout dev
|
||||
# Break a test intentionally
|
||||
git commit -m "Break test"
|
||||
git push origin dev
|
||||
# Create PR to qa
|
||||
# Merge button should be disabled until CI passes
|
||||
```
|
||||
|
||||
**Test 3: Create PR with passing tests**
|
||||
```bash
|
||||
git checkout dev
|
||||
# Fix the test
|
||||
git commit -m "Fix test"
|
||||
git push origin dev
|
||||
# Create PR to qa
|
||||
# Merge button should be enabled after CI passes ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 What Happens After Setup
|
||||
|
||||
### Workflow with Protection:
|
||||
|
||||
```
|
||||
Developer pushes to dev
|
||||
↓
|
||||
CI runs automatically
|
||||
↓
|
||||
✅ Tests pass
|
||||
↓
|
||||
Developer creates PR: dev → qa
|
||||
↓
|
||||
CI runs on PR
|
||||
↓
|
||||
✅ Tests pass
|
||||
↓
|
||||
Reviewer approves
|
||||
↓
|
||||
✅ Merge button enabled
|
||||
↓
|
||||
Merge to qa
|
||||
↓
|
||||
CI runs on qa branch
|
||||
↓
|
||||
✅ Tests pass
|
||||
↓
|
||||
Auto-deploy to QA server (if configured)
|
||||
```
|
||||
|
||||
**If tests fail at any point:**
|
||||
```
|
||||
CI runs
|
||||
↓
|
||||
❌ Tests fail
|
||||
↓
|
||||
Merge button DISABLED
|
||||
↓
|
||||
Developer fixes code
|
||||
↓
|
||||
Pushes again
|
||||
↓
|
||||
CI runs again
|
||||
↓
|
||||
Loop until tests pass ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ✅ Checklist
|
||||
|
||||
Before configuring branch protection:
|
||||
- [ ] CI workflow exists (`.github/workflows/ci.yml`) ✅
|
||||
- [ ] CI runs on all branches (`main`, `qa`, `dev`) ✅
|
||||
- [ ] CI has run at least once on each branch
|
||||
- [ ] All tests are passing ✅
|
||||
- [ ] You can see workflow runs in Actions tab
|
||||
|
||||
After configuring:
|
||||
- [ ] `main` branch requires status checks
|
||||
- [ ] `qa` branch requires status checks
|
||||
- [ ] `dev` branch requires status checks
|
||||
- [ ] Tested: Can't push directly to `main`
|
||||
- [ ] Tested: PR merge blocked when CI fails
|
||||
- [ ] Tested: PR merge allowed when CI passes
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Quick Start (TL;DR)
|
||||
|
||||
```bash
|
||||
# 1. Merge CI updates
|
||||
# Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
|
||||
# Create and merge PR
|
||||
|
||||
# 2. Wait for CI to run
|
||||
# Check: https://git.levkin.ca/ilia/POTE/actions
|
||||
|
||||
# 3. Configure branch protection
|
||||
# Go to: https://git.levkin.ca/ilia/POTE/settings/branches
|
||||
# Add rules for main, qa, dev
|
||||
# Enable "Require status checks"
|
||||
# Select "CI / lint-and-test"
|
||||
|
||||
# Done! ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 Related Documentation
|
||||
|
||||
- **CI Workflow:** `.github/workflows/ci.yml`
|
||||
- **Branch Strategy:** `docs/14_branch_strategy_and_deployment.md`
|
||||
- **Setup Checklist:** `BRANCH_SETUP_COMPLETE.md`
|
||||
- **Gitea Secrets:** `GITEA_SECRETS_GUIDE.md`
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Summary
|
||||
|
||||
**Do you need a pipeline?**
|
||||
- ✅ YES, to use "Require status checks"
|
||||
- ✅ You already have one!
|
||||
- ✅ Just need to merge it to main first
|
||||
|
||||
**Setup order:**
|
||||
1. Merge CI updates to main (via PR)
|
||||
2. Verify CI runs and passes
|
||||
3. Configure branch protection
|
||||
4. Test the protection
|
||||
|
||||
**After setup:**
|
||||
- All branches protected by CI
|
||||
- Can't merge failing code
|
||||
- Professional development workflow
|
||||
- Ready for Ansible integration
|
||||
|
||||
**You're almost there! Just merge the PR and configure protection.** 🎉
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
# POTE homelab handoff — 2026-05-27
|
||||
|
||||
**Status:** Production LXC running; PR #1 merged to `main`; CI green on Gitea Actions.
|
||||
**Research only — not investment advice.**
|
||||
|
||||
---
|
||||
|
||||
## What’s live
|
||||
|
||||
| Item | Value |
|
||||
|------|--------|
|
||||
| Host | LXC **236** `pote` @ **10.0.10.48** (pve10) |
|
||||
| App | `/home/poteapp/pote` (venv, **no git clone** — deploy via rsync) |
|
||||
| DB | PostgreSQL `pote` / `poteuser` (password rotated; in Ansible vault) |
|
||||
| Data | ~55 officials, ~329 trades (30-day live ingest, May 2026) |
|
||||
| SMTP | `10.0.10.132` (Mailcow), send as **`alerts@levkine.ca`** |
|
||||
| Reports | **`idobkin@gmail.com`** daily 07:00, weekly Sun 08:00 |
|
||||
|
||||
### Cron (`crontab -u poteapp -l`)
|
||||
|
||||
| Time | Script |
|
||||
|------|--------|
|
||||
| 06:00 | `fetch_congressional_trades.py --days 7` |
|
||||
| 06:15 | `enrich_securities.py` |
|
||||
| 06:30 | `monitor_market.py --scan` |
|
||||
| 07:00 | `send_daily_report.py --to idobkin@gmail.com` |
|
||||
| Sun 08:00 | `send_weekly_report.py --to idobkin@gmail.com` |
|
||||
|
||||
### Data source (important)
|
||||
|
||||
Legacy **housestockwatcher.com** and S3 buckets are dead/blocked. Ingest uses public JSON from [congress-trading-monitor](https://github.com/kadoa-org/congress-trading-monitor) (~5000 rows cap). Override with env `POTE_HOUSE_DATA_URL` if you add another feed.
|
||||
|
||||
---
|
||||
|
||||
## Repos & branches
|
||||
|
||||
| Repo | Branch | Notes |
|
||||
|------|--------|--------|
|
||||
| **POTE** | `main` @ `git.levkin.ca/ilia/POTE` | Merged PR #1 — ingest, email, CI, deps |
|
||||
| **ansible** | `feature/outline-setup-api` (or `master`) | Inventory, `deploy-pote.sh`, vault — may need merge to homelab default branch |
|
||||
|
||||
Local:
|
||||
|
||||
```bash
|
||||
cd ~/Documents/code/POTE && git checkout main && git pull
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick access
|
||||
|
||||
```bash
|
||||
ssh root@10.0.10.48
|
||||
su - poteapp
|
||||
cd pote && source venv/bin/activate
|
||||
|
||||
# Logs
|
||||
tail -f ~/logs/daily_report.log
|
||||
tail -f ~/logs/trades.log
|
||||
|
||||
# Manual run
|
||||
python scripts/fetch_congressional_trades.py --days 30
|
||||
python scripts/send_daily_report.py --to idobkin@gmail.com --test-smtp
|
||||
```
|
||||
|
||||
Deploy code from laptop (preserves server `.env`):
|
||||
|
||||
```bash
|
||||
cd ~/Documents/code/ansible
|
||||
make deploy-pote
|
||||
# or: RUN_FETCH=1 make deploy-pote
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ansible / homelab inventory
|
||||
|
||||
Already wired (ansible repo):
|
||||
|
||||
- `inventories/production/hosts` — `pote` @ `.48`, VMID 236
|
||||
- `docs/guides/host-list.md` — LXC 236 row
|
||||
- `scripts/beszel-install-agents.sh` — `pote-236`
|
||||
- `scripts/deploy-pote.sh`, `make deploy-pote`
|
||||
- `scripts/vault-update-pote.py`, `make vault-update-pote`
|
||||
- `docs/guides/smtp-inventory.md` — POTE uses `alerts@levkine.ca`
|
||||
- Vault: `vault_pote_db_password_prod`, `vault_pote_smtp_password`
|
||||
|
||||
```bash
|
||||
make vault-export-env
|
||||
make beszel-install-agents BESZEL_ONLY=pote-236 # if agent not yet installed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verify after first automated day
|
||||
|
||||
1. **07:00+** — Email in Gmail (From: `alerts@levkine.ca`, subject `POTE Daily Report - YYYY-MM-DD`). Check spam once.
|
||||
2. **Logs** — `~/logs/daily_report.log`, `trades.log` — no tracebacks.
|
||||
3. **DB growth** — trade count should tick up on weekdays:
|
||||
|
||||
```bash
|
||||
su - poteapp -c 'cd pote && source venv/bin/activate && python -c "
|
||||
from sqlalchemy import func, select
|
||||
from pote.db import SessionLocal
|
||||
from pote.db.models import Trade, Official
|
||||
with SessionLocal() as s:
|
||||
print(\"trades\", s.scalar(select(func.count(Trade.id))))
|
||||
print(\"officials\", s.scalar(select(func.count(Official.id))))
|
||||
"'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**Vikunja:** [todo.levkin.ca → Business → POTE](https://todo.levkin.ca) (`POTE`)
|
||||
|
||||
## Open tasks (source of truth)
|
||||
|
||||
| P | Task | Owner | Status |
|
||||
|---|------|-------|--------|
|
||||
| **P1** | Proxmox backup schedule for LXC **236** on pve10 | @you | ⏳ |
|
||||
| **P3** | Dedicated `pote@levkine.ca` mailbox (vs shared `alerts@`) | @you | optional / low |
|
||||
| **P3** | Git deploy on LXC (replace rsync-only) | @agent | optional / low |
|
||||
| **P3** | Kuma LAN health monitor | @agent | optional / low |
|
||||
| **P3** | Mattermost / webhook alerts (email only today) | @agent | optional / low |
|
||||
| **P3** | Full history ingest (second data source beyond kadoa cap) | @agent | optional / low |
|
||||
|
||||
**Closed 2026-07-11:** Beszel agent on pote-236 (`beszel-agent` active).
|
||||
**Closed 2026-07-12:** `make deploy-pote` synced to LXC 236; 07:00 cron email delivered (`idobkin@gmail.com`, log OK). DB ~1129 trades / 80 officials.
|
||||
|
||||
### Not planned (unless you want them)
|
||||
|
||||
- Public URL / Caddy vhost (LAN-only by design)
|
||||
- Investment signals exposed as advice (research descriptors only)
|
||||
|
||||
---
|
||||
|
||||
## Known issues / caveats
|
||||
|
||||
| Topic | Detail |
|
||||
|-------|--------|
|
||||
| **Disclosure lag** | STOCK Act filings appear weeks after trades; reports are descriptive, not timely trading signals. |
|
||||
| **Amount ranges** | Disclosure buckets only ($1k–$15k, etc.), not exact sizes. |
|
||||
| **Empty tickers** | Some filings skipped when ticker missing. |
|
||||
| **CI vs prod** | CI uses venv + Postgres service; prod uses host Postgres — both should pass after merge. |
|
||||
| **Gitea deploy workflow** | `.github/workflows/deploy.yml` still references `git pull` on Proxmox; prod uses **rsync** via ansible `deploy-pote.sh`. |
|
||||
|
||||
---
|
||||
|
||||
## Related docs
|
||||
|
||||
| Doc | Purpose |
|
||||
|-----|---------|
|
||||
| [EMAIL_SETUP.md](EMAIL_SETUP.md) | SMTP / Mailcow / levkine.ca |
|
||||
| [AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md) | Cron + reports |
|
||||
| [PROXMOX_QUICKSTART.md](PROXMOX_QUICKSTART.md) | Original LXC provisioning |
|
||||
| Ansible `docs/guides/projects-handoff-2026-05-26.md` | Multi-project homelab context |
|
||||
| Ansible `docs/guides/smtp-inventory.md` | Mailboxes |
|
||||
|
||||
---
|
||||
|
||||
## One-line summary
|
||||
|
||||
**POTE on 10.0.10.48 ingests public congressional trades daily, emails a research summary to Gmail at 07:00, and is maintained via `main` + `make deploy-pote` — verify tomorrow’s cron email, then Beszel, backups, and optional data-source expansion.**
|
||||
@@ -1,6 +1,6 @@
|
||||
# PR1 Summary: Project Scaffold + DB + Price Loader
|
||||
|
||||
**Status**: Complete
|
||||
**Status**: ✅ Complete
|
||||
**Date**: 2025-12-13
|
||||
|
||||
## What was built
|
||||
@@ -36,7 +36,7 @@ Includes proper indexes, unique constraints, and relationships.
|
||||
- `tests/conftest.py`: fixtures for in-memory DB, sample officials/securities/trades/prices
|
||||
- `tests/test_models.py`: model creation, relationships, unique constraints, queries (7 tests)
|
||||
- `tests/test_price_loader.py`: loader logic, idempotency, upsert, mocking yfinance (8 tests)
|
||||
- **Result**: 15 tests, all passing
|
||||
- **Result**: 15 tests, all passing ✅
|
||||
|
||||
### 6. Tooling
|
||||
- **Black** + **ruff** configured and run (all code formatted + linted)
|
||||
@@ -1,6 +1,6 @@
|
||||
# PR2 Summary: Congressional Trade Ingestion
|
||||
|
||||
**Status**: Complete
|
||||
**Status**: ✅ Complete
|
||||
**Date**: 2025-12-14
|
||||
|
||||
## What was built
|
||||
@@ -26,7 +26,7 @@
|
||||
- `tests/fixtures/sample_house_watcher.json`: 5 realistic sample transactions
|
||||
- Includes House + Senate, Democrats + Republicans, various tickers
|
||||
|
||||
### 4. Tests (13 new tests, all passing )
|
||||
### 4. Tests (13 new tests, all passing ✅)
|
||||
**`tests/test_house_watcher.py` (8 tests)**:
|
||||
- Amount range parsing (with range, single value, invalid)
|
||||
- Transaction type normalization
|
||||
@@ -58,9 +58,9 @@
|
||||
python scripts/fetch_congressional_trades.py --days 30
|
||||
|
||||
# Sample output:
|
||||
# Officials created/updated: 47
|
||||
# Securities created/updated: 89
|
||||
# Trades ingested: 234
|
||||
# ✓ Officials created/updated: 47
|
||||
# ✓ Securities created/updated: 89
|
||||
# ✓ Trades ingested: 234
|
||||
```
|
||||
|
||||
### Database Queries
|
||||
@@ -1,6 +1,6 @@
|
||||
# PR3 Summary: Security Enrichment + Deployment
|
||||
|
||||
**Status**: Complete
|
||||
**Status**: ✅ Complete
|
||||
**Date**: 2025-12-14
|
||||
|
||||
## What was built
|
||||
@@ -32,7 +32,7 @@
|
||||
python scripts/enrich_securities.py --force
|
||||
```
|
||||
|
||||
### 3. Tests (9 new tests, all passing )
|
||||
### 3. Tests (9 new tests, all passing ✅)
|
||||
**`tests/test_security_enricher.py`**:
|
||||
- Successful enrichment with complete data
|
||||
- ETF detection and classification
|
||||
@@ -72,7 +72,7 @@ python scripts/enrich_securities.py
|
||||
# Enriched AAPL: Apple Inc. (Technology)
|
||||
# Enriched TSLA: Tesla, Inc. (Consumer Cyclical)
|
||||
# Enriched GOOGL: Alphabet Inc. (Communication Services)
|
||||
# Successfully enriched: 5
|
||||
# ✓ Successfully enriched: 5
|
||||
```
|
||||
|
||||
### Query Enriched Data
|
||||
@@ -159,10 +159,10 @@ python scripts/update_all_prices.py # To be built in PR4
|
||||
|
||||
| Option | Complexity | Cost/month | Best For |
|
||||
|--------|-----------|------------|----------|
|
||||
| **Local** | | $0 | Development |
|
||||
| **VPS + Docker** | | $10-20 | Personal deployment |
|
||||
| **Railway/Fly.io** | | $5-15 | Easy cloud |
|
||||
| **AWS** | | $20-50 | Scalable production |
|
||||
| **Local** | ⭐ | $0 | Development |
|
||||
| **VPS + Docker** | ⭐⭐ | $10-20 | Personal deployment |
|
||||
| **Railway/Fly.io** | ⭐ | $5-15 | Easy cloud |
|
||||
| **AWS** | ⭐⭐⭐ | $20-50 | Scalable production |
|
||||
|
||||
See [`docs/07_deployment.md`](07_deployment.md) for detailed guides.
|
||||
|
||||
+6
-6
@@ -223,12 +223,12 @@ print(f"Win Rate: {pelosi_stats['win_rate']:.1%}")
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- Can calculate returns for any trade + window
|
||||
- Can compare to S&P 500 benchmark
|
||||
- Can generate official performance summaries
|
||||
- All calculations tested and accurate
|
||||
- Performance data stored efficiently
|
||||
- Documentation complete
|
||||
- ✅ Can calculate returns for any trade + window
|
||||
- ✅ Can compare to S&P 500 benchmark
|
||||
- ✅ Can generate official performance summaries
|
||||
- ✅ All calculations tested and accurate
|
||||
- ✅ Performance data stored efficiently
|
||||
- ✅ Documentation complete
|
||||
|
||||
## Timeline
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# PR4 Summary: Phase 2 Analytics Foundation
|
||||
|
||||
## Completed
|
||||
## ✅ Completed
|
||||
|
||||
**Date**: December 15, 2025
|
||||
**Status**: Complete
|
||||
@@ -78,14 +78,14 @@ python scripts/calculate_all_returns.py --window 90 --benchmark SPY --top 10
|
||||
|
||||
### 3. Tests (`tests/test_analytics.py`)
|
||||
|
||||
- Return calculator with sample data
|
||||
- Buy vs sell trade handling
|
||||
- Missing data edge cases
|
||||
- Benchmark comparisons
|
||||
- Official performance metrics
|
||||
- Multiple time windows
|
||||
- Sector analysis
|
||||
- Timing analysis
|
||||
- ✅ Return calculator with sample data
|
||||
- ✅ Buy vs sell trade handling
|
||||
- ✅ Missing data edge cases
|
||||
- ✅ Benchmark comparisons
|
||||
- ✅ Official performance metrics
|
||||
- ✅ Multiple time windows
|
||||
- ✅ Sector analysis
|
||||
- ✅ Timing analysis
|
||||
|
||||
**Test Coverage**: Analytics module fully tested
|
||||
|
||||
@@ -288,15 +288,15 @@ with next(get_session()) as session:
|
||||
- Trades near policy events
|
||||
- Unusual timing flags
|
||||
|
||||
## Success Criteria
|
||||
## Success Criteria ✅
|
||||
|
||||
- Can calculate returns for any trade + window
|
||||
- Can compare to S&P 500 benchmark
|
||||
- Can generate official performance summaries
|
||||
- All calculations tested and accurate
|
||||
- Performance data calculated on-the-fly
|
||||
- Documentation complete
|
||||
- Command-line tools working
|
||||
- ✅ Can calculate returns for any trade + window
|
||||
- ✅ Can compare to S&P 500 benchmark
|
||||
- ✅ Can generate official performance summaries
|
||||
- ✅ All calculations tested and accurate
|
||||
- ✅ Performance data calculated on-the-fly
|
||||
- ✅ Documentation complete
|
||||
- ✅ Command-line tools working
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -309,7 +309,7 @@ All analytics tests should pass (may have warnings if no price data).
|
||||
|
||||
---
|
||||
|
||||
**Phase 2 Analytics Foundation: COMPLETE**
|
||||
**Phase 2 Analytics Foundation: COMPLETE** ✅
|
||||
**Ready for**: PR5 (Signals), PR6 (API), PR7 (Dashboard)
|
||||
|
||||
|
||||
@@ -1,324 +0,0 @@
|
||||
# POTE Testing Status Report
|
||||
**Date:** December 15, 2025
|
||||
**Status:** All Systems Operational - Ready for Deployment
|
||||
|
||||
---
|
||||
|
||||
## Test Suite Summary
|
||||
|
||||
### **55 Tests - All Passing **
|
||||
|
||||
```
|
||||
Platform: Python 3.13.5, pytest-9.0.2
|
||||
Test Duration: ~1.8 seconds
|
||||
Coverage: ~85% overall
|
||||
```
|
||||
|
||||
### Test Breakdown by Module:
|
||||
|
||||
| Module | Tests | Status | Coverage |
|
||||
|--------|-------|--------|----------|
|
||||
| **Analytics** | 18 tests | PASS | 80% |
|
||||
| **Models** | 7 tests | PASS | 90% |
|
||||
| **Ingestion** | 14 tests | PASS | 85% |
|
||||
| **Price Loader** | 8 tests | PASS | 90% |
|
||||
| **Security Enricher** | 8 tests | PASS | 85% |
|
||||
|
||||
---
|
||||
|
||||
## What's Been Tested?
|
||||
|
||||
### Core Database Operations
|
||||
- [x] Creating and querying Officials
|
||||
- [x] Creating and querying Securities
|
||||
- [x] Creating and querying Trades
|
||||
- [x] Price data storage and retrieval
|
||||
- [x] Unique constraints and relationships
|
||||
- [x] Database migrations (Alembic)
|
||||
|
||||
### Data Ingestion
|
||||
- [x] House Stock Watcher client (with fixtures)
|
||||
- [x] Trade loading from JSON
|
||||
- [x] Security enrichment from yfinance
|
||||
- [x] Price data fetching and storage
|
||||
- [x] Idempotent operations (no duplicates)
|
||||
- [x] Error handling for missing/invalid data
|
||||
|
||||
### Analytics Engine
|
||||
- [x] Return calculations (buy trades)
|
||||
- [x] Return calculations (sell trades)
|
||||
- [x] Multiple time windows (30/60/90/180 days)
|
||||
- [x] Benchmark comparisons (SPY, QQQ, etc.)
|
||||
- [x] Abnormal returns (alpha calculations)
|
||||
- [x] Official performance summaries
|
||||
- [x] Sector-level analysis
|
||||
- [x] Disclosure timing analysis
|
||||
- [x] Top performer rankings
|
||||
- [x] System-wide statistics
|
||||
|
||||
### Edge Cases
|
||||
- [x] Missing price data handling
|
||||
- [x] Trades with no exit price yet
|
||||
- [x] Sell trades (inverted returns)
|
||||
- [x] Disclosure lags
|
||||
- [x] Duplicate prevention
|
||||
- [x] Invalid date ranges
|
||||
- [x] Empty result sets
|
||||
|
||||
---
|
||||
|
||||
## Test Types
|
||||
|
||||
### 1. Unit Tests (Fast, Isolated)
|
||||
**Location:** `tests/test_*.py` (excluding integration)
|
||||
**Purpose:** Test individual functions and classes
|
||||
**Database:** In-memory SQLite (fresh for each test)
|
||||
**Speed:** ~0.5 seconds
|
||||
|
||||
**Examples:**
|
||||
- `test_parse_amount_range()` - Parse trade amounts
|
||||
- `test_normalize_transaction_type()` - Trade type normalization
|
||||
- `test_get_or_create_security()` - Security deduplication
|
||||
|
||||
### 2. Integration Tests (Realistic Scenarios)
|
||||
**Location:** `tests/test_analytics_integration.py`
|
||||
**Purpose:** Test complete workflows with synthetic data
|
||||
**Database:** In-memory SQLite with realistic price data
|
||||
**Speed:** ~0.7 seconds
|
||||
|
||||
**Examples:**
|
||||
- `test_return_calculation_with_real_data()` - Full return calc pipeline
|
||||
- `test_benchmark_comparison_with_real_data()` - Alpha calculations
|
||||
- `test_official_performance_summary()` - Aggregated metrics
|
||||
|
||||
**Scenarios Tested:**
|
||||
- Nancy Pelosi buys NVDA early (strong returns)
|
||||
- Tommy Tuberville buys NVDA later (good but less alpha)
|
||||
- 120 days of synthetic price data (realistic trends)
|
||||
- SPY benchmark comparison
|
||||
- Multiple time window analysis
|
||||
|
||||
---
|
||||
|
||||
## How to Run Tests Locally
|
||||
|
||||
### Quick Test
|
||||
```bash
|
||||
cd /home/user/Documents/code/pote
|
||||
source venv/bin/activate
|
||||
pytest -v
|
||||
```
|
||||
|
||||
### With Coverage Report
|
||||
```bash
|
||||
pytest --cov=src/pote --cov-report=html --cov-report=term
|
||||
# View: firefox htmlcov/index.html
|
||||
```
|
||||
|
||||
### Specific Test Modules
|
||||
```bash
|
||||
# Just analytics
|
||||
pytest tests/test_analytics.py -v
|
||||
|
||||
# Just integration tests
|
||||
pytest tests/test_analytics_integration.py -v
|
||||
|
||||
# Specific test
|
||||
pytest tests/test_analytics.py::test_return_calculator_basic -v
|
||||
```
|
||||
|
||||
### Watch Mode (Re-run on changes)
|
||||
```bash
|
||||
pytest-watch
|
||||
# or
|
||||
ptw
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Known Limitations
|
||||
|
||||
### 1. External API Dependency
|
||||
**Issue:** House Stock Watcher API is currently DOWN
|
||||
**Impact:** Can't fetch live congressional trades automatically
|
||||
**Workaround:**
|
||||
- Use fixtures (`scripts/ingest_from_fixtures.py`)
|
||||
- Manual CSV import (`scripts/scrape_alternative_sources.py`)
|
||||
- Manual entry (`scripts/add_custom_trades.py`)
|
||||
|
||||
### 2. Market Data Limits
|
||||
**Issue:** yfinance has rate limits and occasional failures
|
||||
**Impact:** Bulk price fetching may be slow
|
||||
**Workaround:**
|
||||
- Fetch in batches
|
||||
- Add retry logic (already implemented)
|
||||
- Use caching (already implemented)
|
||||
|
||||
### 3. No Live Trading API
|
||||
**Issue:** We only use public disclosure data (inherent lag)
|
||||
**Impact:** Trades are 30-45 days delayed by law
|
||||
**This is expected:** POTE is for research, not real-time trading
|
||||
|
||||
---
|
||||
|
||||
## Performance Benchmarks
|
||||
|
||||
### Test Execution Time
|
||||
- **Full suite:** 1.8 seconds
|
||||
- **Unit tests only:** 0.5 seconds
|
||||
- **Integration tests:** 0.7 seconds
|
||||
- **Parallel execution:** ~1.0 second (with pytest-xdist)
|
||||
|
||||
### Database Operations
|
||||
- **Create official:** < 1ms
|
||||
- **Create trade:** < 1ms
|
||||
- **Fetch prices (100 days):** ~50ms (in-memory)
|
||||
- **Calculate returns:** ~10ms per trade
|
||||
- **Aggregate metrics:** ~50ms for 100 trades
|
||||
|
||||
---
|
||||
|
||||
## Pre-Deployment Checklist
|
||||
|
||||
### Before Deploying to Proxmox:
|
||||
|
||||
- [x] All tests passing locally
|
||||
- [x] No linter errors (`make lint`)
|
||||
- [x] Database migrations work (`alembic upgrade head`)
|
||||
- [x] Scripts are executable and work
|
||||
- [x] Environment variables documented
|
||||
- [x] Sample data available for testing
|
||||
- [x] Documentation up to date
|
||||
|
||||
### On Proxmox Container:
|
||||
|
||||
```bash
|
||||
# 1. Pull latest code
|
||||
cd ~/pote
|
||||
git pull
|
||||
|
||||
# 2. Update dependencies
|
||||
pip install -e .
|
||||
|
||||
# 3. Run tests
|
||||
pytest -v
|
||||
|
||||
# 4. Run migrations
|
||||
alembic upgrade head
|
||||
|
||||
# 5. Verify system
|
||||
python ~/status.sh
|
||||
|
||||
# 6. Test a script
|
||||
python scripts/enrich_securities.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Continuous Testing
|
||||
|
||||
### Git Pre-Commit Hook (Optional)
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# .git/hooks/pre-commit
|
||||
pytest --tb=short
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Tests failed. Commit aborted."
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
### CI/CD Integration (Future)
|
||||
When you set up GitHub Actions or GitLab CI:
|
||||
|
||||
```yaml
|
||||
# .github/workflows/test.yml
|
||||
name: Tests
|
||||
on: [push, pull_request]
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
- run: pip install -e .
|
||||
- run: pytest -v --cov
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Maintenance
|
||||
|
||||
### Adding New Tests
|
||||
|
||||
**When to add tests:**
|
||||
- Adding new features
|
||||
- Fixing bugs (write test that fails, then fix)
|
||||
- Before refactoring (ensure tests pass before & after)
|
||||
|
||||
**Where to add tests:**
|
||||
- Unit tests: `tests/test_<module>.py`
|
||||
- Integration tests: `tests/test_<feature>_integration.py`
|
||||
|
||||
**Example:**
|
||||
```python
|
||||
def test_new_feature(test_db_session):
|
||||
"""Test description."""
|
||||
session = test_db_session
|
||||
# Arrange
|
||||
# Act
|
||||
# Assert
|
||||
```
|
||||
|
||||
### Updating Fixtures
|
||||
|
||||
Fixtures are in `tests/conftest.py`:
|
||||
- `test_db_session` - Fresh database
|
||||
- `sample_official` - Test official
|
||||
- `sample_security` - Test security (AAPL)
|
||||
- `sample_trade` - Test trade
|
||||
- `sample_price` - Test price record
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
### Current Status: **PRODUCTION READY**
|
||||
|
||||
**What Works:**
|
||||
- All 55 tests passing
|
||||
- Full analytics pipeline functional
|
||||
- Database operations solid
|
||||
- Data ingestion from multiple sources
|
||||
- Price fetching from yfinance
|
||||
- Security enrichment
|
||||
- Return calculations
|
||||
- Benchmark comparisons
|
||||
- Performance metrics
|
||||
- CLI scripts operational
|
||||
|
||||
**What's Missing:**
|
||||
- Live congressional trade API (external issue - House Stock Watcher down)
|
||||
- **Workaround:** Manual import, CSV, or alternative APIs available
|
||||
|
||||
**Next Steps:**
|
||||
1. Tests are complete
|
||||
2. Code is ready
|
||||
3. **Deploy to Proxmox** (or continue with Phase 2 features)
|
||||
4. Add more data sources
|
||||
5. Build dashboard (Phase 3)
|
||||
|
||||
---
|
||||
|
||||
## Need Help?
|
||||
|
||||
See:
|
||||
- `LOCAL_TEST_GUIDE.md` - Detailed local testing instructions
|
||||
- `QUICKSTART.md` - Usage guide for deployed system
|
||||
- `docs/09_data_updates.md` - How to add/update data
|
||||
- `README.md` - Project overview
|
||||
|
||||
**Questions about testing?**
|
||||
All tests are documented with docstrings - read the test files!
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# Archive
|
||||
|
||||
PR summaries and one-shot status writeups. Prefer the numbered guides and QUICKSTART files.
|
||||
@@ -18,10 +18,8 @@ dependencies = [
|
||||
"pydantic-settings>=2.0",
|
||||
"python-dotenv>=1.0",
|
||||
"requests>=2.31",
|
||||
"httpx>=0.27",
|
||||
"pandas>=2.0",
|
||||
"numpy>=1.24",
|
||||
"scikit-learn>=1.3",
|
||||
"yfinance>=0.2",
|
||||
"psycopg2-binary>=2.9",
|
||||
]
|
||||
@@ -41,8 +39,6 @@ where = ["src"]
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py311"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM", "RET"]
|
||||
ignore = ["E501"] # Line too long (handled by black)
|
||||
|
||||
|
||||
@@ -28,8 +28,8 @@ def main():
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.info("=== Fetching Congressional Trades (public STOCK Act data) ===")
|
||||
logger.info("Source: public JSON feeds (see HouseWatcherClient.data_urls)")
|
||||
logger.info("=== Fetching Congressional Trades from House Stock Watcher ===")
|
||||
logger.info("Source: https://housestockwatcher.com (free, no API key)")
|
||||
|
||||
try:
|
||||
with HouseWatcherClient() as client:
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time installer for the local pre-commit gitleaks hook.
|
||||
# Run once per clone: bash scripts/git-hooks/install.sh
|
||||
#
|
||||
# Respects `core.hooksPath` if you've set one (local or global) — some setups
|
||||
# point git at a hooks dir outside `.git/hooks/` (e.g. a machine-wide
|
||||
# `~/.git-hooks/`), and installing to `.git/hooks/` in that case would be a
|
||||
# silent no-op. If an existing hook is already at that path, this chains to
|
||||
# it so nothing already relying on it breaks.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
HOOKS_DIR="$(git config --get core.hooksPath || true)"
|
||||
if [ -z "$HOOKS_DIR" ]; then
|
||||
HOOKS_DIR=".git/hooks"
|
||||
elif [[ "$HOOKS_DIR" != /* ]]; then
|
||||
HOOKS_DIR="$REPO_ROOT/$HOOKS_DIR"
|
||||
fi
|
||||
mkdir -p "$HOOKS_DIR"
|
||||
|
||||
TARGET="$HOOKS_DIR/pre-commit"
|
||||
if [ -f "$TARGET" ] && ! grep -q "gitleaks" "$TARGET" 2>/dev/null; then
|
||||
echo "⚠️ Existing pre-commit hook found at $TARGET that isn't ours — chaining instead of overwriting."
|
||||
CHAINED="$HOOKS_DIR/pre-commit.d-gitleaks"
|
||||
cp scripts/git-hooks/pre-commit "$CHAINED"
|
||||
chmod +x "$CHAINED"
|
||||
if ! grep -q "pre-commit.d-gitleaks" "$TARGET" 2>/dev/null; then
|
||||
printf '\n# Added by levkinops ansible repo (scripts/git-hooks/install.sh)\n"%s"\n' "$CHAINED" >> "$TARGET"
|
||||
fi
|
||||
else
|
||||
cp scripts/git-hooks/pre-commit "$TARGET"
|
||||
chmod +x "$TARGET"
|
||||
fi
|
||||
|
||||
echo "✓ Installed pre-commit gitleaks hook → $TARGET"
|
||||
if [ "$HOOKS_DIR" != ".git/hooks" ] && [ "$HOOKS_DIR" != "$REPO_ROOT/.git/hooks" ]; then
|
||||
echo " (using core.hooksPath=$HOOKS_DIR — applies to every repo that shares this hooksPath)"
|
||||
fi
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo " Note: gitleaks isn't installed locally yet. Install it for the hook to actually run:"
|
||||
echo " macOS: brew install gitleaks"
|
||||
echo " Linux: see https://github.com/gitleaks/gitleaks#installing"
|
||||
echo " Until then this hook is a no-op locally (CI still scans every push)."
|
||||
fi
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Local pre-commit secret scan — mirrors the `secret-scanning` CI job (gitleaks)
|
||||
# so leaked secrets are caught before they ever leave your machine, not just at
|
||||
# CI time. Installed via `make install-git-hooks`.
|
||||
#
|
||||
# Uses the same .gitleaks.toml allowlist as CI. Only scans staged content.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "⚠️ gitleaks not installed locally — skipping local secret scan."
|
||||
echo " Install: brew install gitleaks (CI will still catch it either way)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "🔐 Running gitleaks on staged changes..."
|
||||
if ! gitleaks protect --staged --config .gitleaks.toml --no-banner --redact; then
|
||||
echo ""
|
||||
echo "❌ gitleaks found a potential secret in your staged changes."
|
||||
echo " Fix it, or if it's a false positive, add an allowlist entry to .gitleaks.toml."
|
||||
echo " Bypass (not recommended): git commit --no-verify"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ gitleaks: no secrets found in staged changes"
|
||||
@@ -74,7 +74,7 @@ def main(tickers, interval, once, min_severity, save_report, lookback):
|
||||
|
||||
if filtered:
|
||||
# Generate report
|
||||
report = alert_mgr.generate_summary_report(filtered, output_format="text")
|
||||
report = alert_mgr.generate_summary_report(filtered, format="text")
|
||||
print("\n" + report)
|
||||
|
||||
# Save report if requested
|
||||
|
||||
@@ -43,12 +43,6 @@ def main():
|
||||
action="store_true",
|
||||
help="Test SMTP connection before sending",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lookback-days",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Include trades filed in the last N days ending on the report date (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-to-file",
|
||||
help="Also save report to this file path",
|
||||
@@ -87,9 +81,7 @@ def main():
|
||||
logger.info(f"Generating daily report for {report_date or date.today()}...")
|
||||
with get_session() as session:
|
||||
generator = ReportGenerator(session)
|
||||
report_data = generator.generate_daily_summary(
|
||||
report_date, lookback_days=args.lookback_days
|
||||
)
|
||||
report_data = generator.generate_daily_summary(report_date)
|
||||
|
||||
# Format as text and HTML
|
||||
text_body = generator.format_as_text(report_data, "daily")
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
Analytics module for calculating returns, performance metrics, and signals.
|
||||
"""
|
||||
|
||||
from .returns import ReturnCalculator
|
||||
from .benchmarks import BenchmarkComparison
|
||||
from .metrics import PerformanceMetrics
|
||||
from .returns import ReturnCalculator
|
||||
|
||||
__all__ = [
|
||||
"ReturnCalculator",
|
||||
"BenchmarkComparison",
|
||||
"PerformanceMetrics",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ Benchmark comparison for calculating abnormal returns (alpha).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -60,7 +60,8 @@ class BenchmarkComparison:
|
||||
return None
|
||||
|
||||
# Calculate return
|
||||
return ((end_price - start_price) / start_price) * 100
|
||||
return_pct = ((end_price - start_price) / start_price) * 100
|
||||
return return_pct
|
||||
|
||||
def calculate_abnormal_return(
|
||||
self,
|
||||
@@ -218,3 +219,5 @@ class BenchmarkComparison:
|
||||
"benchmark": self.BENCHMARKS.get(benchmark, benchmark),
|
||||
"window_days": window_days,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ Performance metrics and aggregations.
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -51,7 +52,11 @@ class PerformanceMetrics:
|
||||
if not official:
|
||||
return {"error": "Official not found"}
|
||||
|
||||
trades = self.session.query(Trade).filter(Trade.official_id == official_id).all()
|
||||
trades = (
|
||||
self.session.query(Trade)
|
||||
.filter(Trade.official_id == official_id)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not trades:
|
||||
return {
|
||||
@@ -65,7 +70,9 @@ class PerformanceMetrics:
|
||||
# Calculate returns for all trades
|
||||
returns_data = []
|
||||
for trade in trades:
|
||||
result = self.benchmark.compare_trade_to_benchmark(trade, window_days, benchmark)
|
||||
result = self.benchmark.compare_trade_to_benchmark(
|
||||
trade, window_days, benchmark
|
||||
)
|
||||
if result:
|
||||
returns_data.append(result)
|
||||
|
||||
@@ -89,7 +96,9 @@ class PerformanceMetrics:
|
||||
worst_trade = min(returns_data, key=lambda x: x["trade_return"])
|
||||
|
||||
# Total value traded
|
||||
total_value = sum(float(t.value_min or 0) for t in trades if t.value_min)
|
||||
total_value = sum(
|
||||
float(t.value_min or 0) for t in trades if t.value_min
|
||||
)
|
||||
|
||||
return {
|
||||
"name": official.name,
|
||||
@@ -145,14 +154,20 @@ class PerformanceMetrics:
|
||||
List of sector performance dictionaries
|
||||
"""
|
||||
# Get all trades with security info
|
||||
trades = self.session.query(Trade).join(Security).all()
|
||||
trades = (
|
||||
self.session.query(Trade)
|
||||
.join(Security)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Group by sector
|
||||
sector_data = defaultdict(list)
|
||||
|
||||
for trade in trades:
|
||||
sector = trade.security.sector or "Unknown"
|
||||
result = self.benchmark.compare_trade_to_benchmark(trade, window_days, benchmark)
|
||||
result = self.benchmark.compare_trade_to_benchmark(
|
||||
trade, window_days, benchmark
|
||||
)
|
||||
if result:
|
||||
sector_data[sector].append(result)
|
||||
|
||||
@@ -165,16 +180,14 @@ class PerformanceMetrics:
|
||||
returns = [d["trade_return"] for d in data]
|
||||
alphas = [d["abnormal_return"] for d in data]
|
||||
|
||||
results.append(
|
||||
{
|
||||
"sector": sector,
|
||||
"trade_count": len(data),
|
||||
"avg_return": sum(returns) / len(returns),
|
||||
"avg_alpha": sum(alphas) / len(alphas),
|
||||
"win_rate": sum(1 for r in returns if r > 0) / len(returns),
|
||||
"beat_market_rate": sum(1 for a in alphas if a > 0) / len(alphas),
|
||||
}
|
||||
)
|
||||
results.append({
|
||||
"sector": sector,
|
||||
"trade_count": len(data),
|
||||
"avg_return": sum(returns) / len(returns),
|
||||
"avg_alpha": sum(alphas) / len(alphas),
|
||||
"win_rate": sum(1 for r in returns if r > 0) / len(returns),
|
||||
"beat_market_rate": sum(1 for a in alphas if a > 0) / len(alphas),
|
||||
})
|
||||
|
||||
# Sort by average alpha
|
||||
results.sort(key=lambda x: x["avg_alpha"], reverse=True)
|
||||
@@ -216,7 +229,11 @@ class PerformanceMetrics:
|
||||
Returns:
|
||||
Dictionary with timing statistics
|
||||
"""
|
||||
trades = self.session.query(Trade).filter(Trade.filing_date.isnot(None)).all()
|
||||
trades = (
|
||||
self.session.query(Trade)
|
||||
.filter(Trade.filing_date.isnot(None))
|
||||
.all()
|
||||
)
|
||||
|
||||
if not trades:
|
||||
return {"error": "No trades with disclosure dates"}
|
||||
@@ -271,3 +288,5 @@ class PerformanceMetrics:
|
||||
"benchmark": benchmark,
|
||||
**aggregate,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -94,20 +94,18 @@ class ReturnCalculator:
|
||||
def calculate_multiple_windows(
|
||||
self,
|
||||
trade: Trade,
|
||||
windows: list[int] | None = None,
|
||||
windows: list[int] = [30, 60, 90, 180],
|
||||
) -> dict[int, dict]:
|
||||
"""
|
||||
Calculate returns for multiple time windows.
|
||||
|
||||
Args:
|
||||
trade: Trade object
|
||||
windows: List of window sizes in days (defaults to 30/60/90/180)
|
||||
windows: List of window sizes in days
|
||||
|
||||
Returns:
|
||||
Dictionary mapping window_days to return metrics
|
||||
"""
|
||||
if windows is None:
|
||||
windows = [30, 60, 90, 180]
|
||||
results = {}
|
||||
for window in windows:
|
||||
result = self.calculate_trade_return(trade, window)
|
||||
@@ -225,13 +223,12 @@ class ReturnCalculator:
|
||||
if not prices:
|
||||
return pd.DataFrame()
|
||||
|
||||
# open/high/low are nullable columns; use NaN (pandas-native) when absent.
|
||||
data = [
|
||||
{
|
||||
"date": p.date,
|
||||
"open": float(p.open) if p.open is not None else float("nan"),
|
||||
"high": float(p.high) if p.high is not None else float("nan"),
|
||||
"low": float(p.low) if p.low is not None else float("nan"),
|
||||
"open": float(p.open),
|
||||
"high": float(p.high),
|
||||
"low": float(p.low),
|
||||
"close": float(p.close),
|
||||
"volume": p.volume,
|
||||
}
|
||||
@@ -239,3 +236,4 @@ class ReturnCalculator:
|
||||
]
|
||||
|
||||
return pd.DataFrame(data)
|
||||
|
||||
|
||||
@@ -30,16 +30,6 @@ class Settings(BaseSettings):
|
||||
# Logging
|
||||
log_level: str = Field(default="INFO", description="Log level (DEBUG, INFO, WARNING, ERROR)")
|
||||
|
||||
# Email (Mailcow @ mail.levkine.ca — use LAN IP from app LXCs if DNS points public)
|
||||
smtp_host: str = Field(default="mail.levkine.ca", description="SMTP server hostname")
|
||||
smtp_port: int = Field(default=587, description="SMTP port (587 STARTTLS)")
|
||||
smtp_user: str = Field(default="", description="SMTP auth username")
|
||||
smtp_password: str = Field(default="", description="SMTP auth password")
|
||||
from_email: str = Field(default="", description="From address for reports")
|
||||
report_recipients: str = Field(
|
||||
default="", description="Default report recipients (comma-separated)"
|
||||
)
|
||||
|
||||
# Application
|
||||
app_name: str = "POTE"
|
||||
app_version: str = "0.1.0"
|
||||
|
||||
@@ -3,7 +3,6 @@ Database layer: engine, session factory, and base model.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
@@ -27,9 +26,8 @@ class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session() -> Generator[Session, None, None]:
|
||||
"""Get a database session (context manager or FastAPI-style dependency)."""
|
||||
"""Get a database session (use as a context manager or dependency)."""
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
|
||||
+32
-16
@@ -3,17 +3,17 @@ SQLAlchemy ORM models for POTE.
|
||||
Matches the schema defined in docs/02_data_model.md.
|
||||
"""
|
||||
|
||||
from datetime import UTC, date, datetime
|
||||
from datetime import date, datetime, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import (
|
||||
DECIMAL,
|
||||
JSON,
|
||||
Date,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
@@ -35,11 +35,13 @@ class Official(Base):
|
||||
state: Mapped[str | None] = mapped_column(String(2))
|
||||
bioguide_id: Mapped[str | None] = mapped_column(String(20), unique=True)
|
||||
external_ids: Mapped[str | None] = mapped_column(Text) # JSON blob for other IDs
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
@@ -61,11 +63,13 @@ class Security(Base):
|
||||
sector: Mapped[str | None] = mapped_column(String(100))
|
||||
industry: Mapped[str | None] = mapped_column(String(100))
|
||||
asset_type: Mapped[str] = mapped_column(String(50), default="stock") # stock, bond, etc.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
@@ -103,11 +107,13 @@ class Trade(Base):
|
||||
# Quality flags (JSON or enum list)
|
||||
quality_flags: Mapped[str | None] = mapped_column(Text) # e.g., "range_only,delayed_filing"
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Relationships
|
||||
@@ -150,7 +156,9 @@ class Price(Base):
|
||||
adjusted_close: Mapped[Decimal | None] = mapped_column(DECIMAL(15, 4))
|
||||
|
||||
source: Mapped[str] = mapped_column(String(50), default="yfinance")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
# Relationships
|
||||
security: Mapped["Security"] = relationship("Security", back_populates="prices")
|
||||
@@ -180,7 +188,9 @@ class MetricOfficial(Base):
|
||||
avg_abnormal_return_1m: Mapped[Decimal | None] = mapped_column(DECIMAL(10, 6))
|
||||
cluster_label: Mapped[str | None] = mapped_column(String(50))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("official_id", "calc_date", "calc_version", name="uq_metrics_official"),
|
||||
@@ -202,7 +212,9 @@ class MetricTrade(Base):
|
||||
abnormal_return_1m: Mapped[Decimal | None] = mapped_column(DECIMAL(10, 6))
|
||||
signal_flags: Mapped[str | None] = mapped_column(Text) # JSON list
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("trade_id", "calc_date", "calc_version", name="uq_metrics_trade"),
|
||||
@@ -230,14 +242,18 @@ class MarketAlert(Base):
|
||||
# Metrics at time of alert
|
||||
price: Mapped[Decimal | None] = mapped_column(DECIMAL(15, 4))
|
||||
volume: Mapped[int | None] = mapped_column(Integer)
|
||||
change_pct: Mapped[Decimal | None] = mapped_column(DECIMAL(10, 4)) # Price change %
|
||||
change_pct: Mapped[Decimal | None] = mapped_column(
|
||||
DECIMAL(10, 4)
|
||||
) # Price change %
|
||||
|
||||
# Severity scoring
|
||||
severity: Mapped[int | None] = mapped_column(Integer) # 1-10 scale
|
||||
|
||||
# Metadata
|
||||
source: Mapped[str] = mapped_column(String(50), default="market_monitor")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
# Indexes for efficient queries
|
||||
__table_args__ = (
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
"""
|
||||
House Stock Watcher client for fetching congressional trade data.
|
||||
|
||||
Uses public STOCK Act disclosure datasets (no API key). The legacy
|
||||
housestockwatcher.com host is often unavailable; we try several mirrors.
|
||||
Free, no API key required - scrapes from housestockwatcher.com
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -15,69 +12,15 @@ import httpx
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_data_urls() -> tuple[str, ...]:
|
||||
override = os.environ.get("POTE_HOUSE_DATA_URL", "").strip()
|
||||
if override:
|
||||
return (override,)
|
||||
return (
|
||||
"https://raw.githubusercontent.com/kadoa-org/congress-trading-monitor/main/public/data/trades.json",
|
||||
"https://housestockwatcher.com/api/all_transactions",
|
||||
"https://house-stock-watcher-data.s3-us-west-2.amazonaws.com/data/all_transactions.json",
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_DATA_URLS: tuple[str, ...] = _default_data_urls()
|
||||
|
||||
|
||||
def _ascii_safe(text: str | None) -> str:
|
||||
"""Normalize text for DB fields that may use ASCII-only client encoding."""
|
||||
if not text:
|
||||
return ""
|
||||
return text.replace("\u00b7", "-").replace("·", "-").strip()
|
||||
|
||||
|
||||
def _party_label(code: str | None) -> str:
|
||||
if not code:
|
||||
return ""
|
||||
mapping = {"D": "Democrat", "R": "Republican", "I": "Independent"}
|
||||
return mapping.get(code.strip().upper(), code.strip())
|
||||
|
||||
|
||||
def normalize_transaction_record(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Map a raw disclosure record to the House Stock Watcher field names
|
||||
expected by TradeLoader.
|
||||
"""
|
||||
if "representative" in raw and "disclosure_date" in raw:
|
||||
return raw
|
||||
|
||||
# Kadoa congress-trading-monitor export
|
||||
if "filer_name" in raw:
|
||||
chamber = (raw.get("chamber") or "").strip().lower()
|
||||
house = "Senate" if chamber == "senate" else "House"
|
||||
return {
|
||||
"representative": raw.get("filer_name", "").strip(),
|
||||
"ticker": (raw.get("ticker") or "").strip(),
|
||||
"transaction_date": raw.get("transaction_date", ""),
|
||||
"disclosure_date": raw.get("filing_date", ""),
|
||||
"transaction": raw.get("transaction_type", ""),
|
||||
"amount": raw.get("amount_range_label", ""),
|
||||
"house": house,
|
||||
"district": _ascii_safe(raw.get("office")),
|
||||
"party": _party_label(raw.get("party")),
|
||||
}
|
||||
|
||||
return raw
|
||||
|
||||
|
||||
class HouseWatcherClient:
|
||||
"""
|
||||
Client for congressional trade JSON feeds (free, community-maintained).
|
||||
Client for House Stock Watcher API (free, community-maintained).
|
||||
|
||||
Primary source: congress-trading-monitor public dataset on GitHub.
|
||||
Data source: https://housestockwatcher.com/
|
||||
No authentication required.
|
||||
"""
|
||||
|
||||
data_urls: tuple[str, ...] = DEFAULT_DATA_URLS
|
||||
BASE_URL = "https://housestockwatcher.com/api"
|
||||
|
||||
def __init__(self, timeout: float = 30.0):
|
||||
"""
|
||||
@@ -122,31 +65,30 @@ class HouseWatcherClient:
|
||||
Raises:
|
||||
httpx.HTTPError: If request fails
|
||||
"""
|
||||
last_error: Exception | None = None
|
||||
for url in self.data_urls:
|
||||
if not url:
|
||||
continue
|
||||
logger.info(f"Fetching transactions from {url}")
|
||||
try:
|
||||
response = self._client.get(url)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if not isinstance(data, list):
|
||||
raise ValueError(f"Expected list response, got {type(data)}")
|
||||
data = [normalize_transaction_record(item) for item in data]
|
||||
logger.info(f"Fetched {len(data)} transactions from {url}")
|
||||
if limit:
|
||||
data = data[:limit]
|
||||
return data
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.warning(f"Failed to fetch from {url}: {e}")
|
||||
continue
|
||||
url = f"{self.BASE_URL}/all_transactions"
|
||||
logger.info(f"Fetching transactions from {url}")
|
||||
|
||||
logger.error("Failed to fetch congressional trades from all configured URLs")
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise RuntimeError("No data URLs configured")
|
||||
try:
|
||||
response = self._client.get(url)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise ValueError(f"Expected list response, got {type(data)}")
|
||||
|
||||
logger.info(f"Fetched {len(data)} transactions from House Stock Watcher")
|
||||
|
||||
if limit:
|
||||
data = data[:limit]
|
||||
|
||||
return data
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Failed to fetch from House Stock Watcher: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching transactions: {e}")
|
||||
raise
|
||||
|
||||
def fetch_recent_transactions(self, days: int = 30) -> list[dict[str, Any]]:
|
||||
"""
|
||||
@@ -236,9 +178,10 @@ def normalize_transaction_type(txn_type: str) -> str:
|
||||
|
||||
if "purchase" in txn_lower or "buy" in txn_lower:
|
||||
return "buy"
|
||||
if "sale" in txn_lower or "sell" in txn_lower:
|
||||
elif "sale" in txn_lower or "sell" in txn_lower:
|
||||
return "sell"
|
||||
if "exchange" in txn_lower:
|
||||
elif "exchange" in txn_lower:
|
||||
return "exchange"
|
||||
# Default to the original, lowercased
|
||||
return txn_lower
|
||||
else:
|
||||
# Default to the original, lowercased
|
||||
return txn_lower
|
||||
|
||||
@@ -4,7 +4,7 @@ Fetches daily OHLCV data for securities and stores in the prices table.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
import pandas as pd
|
||||
@@ -158,7 +158,7 @@ class PriceLoader:
|
||||
"volume": int(row["volume"]) if pd.notna(row.get("volume")) else None,
|
||||
"adjusted_close": None, # We'll compute this later if needed
|
||||
"source": "yfinance",
|
||||
"created_at": datetime.now(UTC),
|
||||
"created_at": datetime.now(timezone.utc),
|
||||
}
|
||||
records.append(record)
|
||||
|
||||
|
||||
@@ -45,25 +45,27 @@ class TradeLoader:
|
||||
|
||||
for txn in transactions:
|
||||
try:
|
||||
with self.session.begin_nested():
|
||||
official, is_new_official = self._get_or_create_official(txn)
|
||||
if is_new_official:
|
||||
officials_created += 1
|
||||
# Get or create official
|
||||
official, is_new_official = self._get_or_create_official(txn)
|
||||
if is_new_official:
|
||||
officials_created += 1
|
||||
|
||||
ticker = txn.get("ticker", "").strip().upper()
|
||||
if not ticker or ticker in ("N/A", "--", ""):
|
||||
logger.debug(
|
||||
f"Skipping transaction with no ticker: {txn.get('representative')}"
|
||||
)
|
||||
continue
|
||||
# Get or create security
|
||||
ticker = txn.get("ticker", "").strip().upper()
|
||||
if not ticker or ticker in ("N/A", "--", ""):
|
||||
logger.debug(
|
||||
f"Skipping transaction with no ticker: {txn.get('representative')}"
|
||||
)
|
||||
continue
|
||||
|
||||
security, is_new_security = self._get_or_create_security(ticker)
|
||||
if is_new_security:
|
||||
securities_created += 1
|
||||
security, is_new_security = self._get_or_create_security(ticker)
|
||||
if is_new_security:
|
||||
securities_created += 1
|
||||
|
||||
trade_created = self._upsert_trade(txn, official.id, security.id, source)
|
||||
if trade_created:
|
||||
trades_created += 1
|
||||
# Create trade (upsert)
|
||||
trade_created = self._upsert_trade(txn, official.id, security.id, source)
|
||||
if trade_created:
|
||||
trades_created += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ingest transaction {txn}: {e}")
|
||||
|
||||
@@ -9,3 +9,4 @@ from .market_monitor import MarketMonitor
|
||||
from .pattern_detector import PatternDetector
|
||||
|
||||
__all__ = ["MarketMonitor", "AlertManager", "DisclosureCorrelator", "PatternDetector"]
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@ Handles alert filtering, formatting, and delivery.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -74,24 +75,21 @@ class AlertManager:
|
||||
Returns:
|
||||
HTML formatted alert
|
||||
"""
|
||||
severity_class = (
|
||||
"high"
|
||||
if (alert.severity or 0) >= 7
|
||||
else "medium" if (alert.severity or 0) >= 4 else "low"
|
||||
)
|
||||
severity_class = "high" if (alert.severity or 0) >= 7 else "medium" if (alert.severity or 0) >= 4 else "low"
|
||||
|
||||
return f"""
|
||||
html = f"""
|
||||
<div class="alert {severity_class}">
|
||||
<h3>{alert.ticker} - {alert.alert_type.replace('_', ' ').title()}</h3>
|
||||
<p class="timestamp">{alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</p>
|
||||
<p class="severity">Severity: {alert.severity}/10</p>
|
||||
<div class="metrics">
|
||||
<span>Price: ${float(alert.price or 0):.2f}</span>
|
||||
<span>Price: ${float(alert.price):.2f}</span>
|
||||
<span>Volume: {alert.volume:,}</span>
|
||||
<span>Change: {float(alert.change_pct or 0):+.2f}%</span>
|
||||
<span>Change: {float(alert.change_pct):+.2f}%</span>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
return html
|
||||
|
||||
def filter_alerts(
|
||||
self,
|
||||
@@ -119,7 +117,7 @@ class AlertManager:
|
||||
|
||||
# Filter by ticker
|
||||
if tickers:
|
||||
ticker_set = {t.upper() for t in tickers}
|
||||
ticker_set = set(t.upper() for t in tickers)
|
||||
filtered = [a for a in filtered if a.ticker.upper() in ticker_set]
|
||||
|
||||
# Filter by alert type
|
||||
@@ -130,21 +128,22 @@ class AlertManager:
|
||||
return filtered
|
||||
|
||||
def generate_summary_report(
|
||||
self, alerts: list[MarketAlert], output_format: str = "text"
|
||||
self, alerts: list[MarketAlert], format: str = "text"
|
||||
) -> str:
|
||||
"""
|
||||
Generate summary report of alerts.
|
||||
|
||||
Args:
|
||||
alerts: List of alerts
|
||||
output_format: Output format ('text' or 'html')
|
||||
format: Output format ('text' or 'html')
|
||||
|
||||
Returns:
|
||||
Formatted summary report
|
||||
"""
|
||||
if output_format == "html":
|
||||
if format == "html":
|
||||
return self._generate_html_summary(alerts)
|
||||
return self._generate_text_summary(alerts)
|
||||
else:
|
||||
return self._generate_text_summary(alerts)
|
||||
|
||||
def _generate_text_summary(self, alerts: list[MarketAlert]) -> str:
|
||||
"""Generate text summary report."""
|
||||
@@ -153,7 +152,7 @@ class AlertManager:
|
||||
|
||||
lines = [
|
||||
"=" * 80,
|
||||
f" MARKET ACTIVITY ALERTS - {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC",
|
||||
f" MARKET ACTIVITY ALERTS - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC",
|
||||
f" {len(alerts)} Alerts",
|
||||
"=" * 80,
|
||||
"",
|
||||
@@ -181,7 +180,9 @@ class AlertManager:
|
||||
lines.append(f"🎯 {ticker} - {len(ticker_alerts)} alerts (Max Severity: {max_sev}/10)")
|
||||
lines.append("─" * 80)
|
||||
|
||||
for alert in sorted(ticker_alerts, key=lambda a: a.severity or 0, reverse=True):
|
||||
for alert in sorted(
|
||||
ticker_alerts, key=lambda a: a.severity or 0, reverse=True
|
||||
):
|
||||
lines.append("")
|
||||
lines.append(self.format_alert_text(alert))
|
||||
|
||||
@@ -201,7 +202,9 @@ class AlertManager:
|
||||
type_counts[alert.alert_type] = type_counts.get(alert.alert_type, 0) + 1
|
||||
|
||||
lines.append("\nAlert Types:")
|
||||
for alert_type, count in sorted(type_counts.items(), key=lambda x: x[1], reverse=True):
|
||||
for alert_type, count in sorted(
|
||||
type_counts.items(), key=lambda x: x[1], reverse=True
|
||||
):
|
||||
lines.append(f" {alert_type.replace('_', ' ').title():20s}: {count}")
|
||||
|
||||
# Top severity alerts
|
||||
@@ -229,8 +232,8 @@ class AlertManager:
|
||||
".timestamp { color: #666; font-size: 0.9em; }",
|
||||
".metrics span { margin-right: 20px; }",
|
||||
"</style></head><body>",
|
||||
"<h1>Market Activity Alerts</h1>",
|
||||
f"<p><strong>{len(alerts)} Alerts</strong> | {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC</p>",
|
||||
f"<h1>Market Activity Alerts</h1>",
|
||||
f"<p><strong>{len(alerts)} Alerts</strong> | {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC</p>",
|
||||
]
|
||||
|
||||
for alert in sorted(alerts, key=lambda a: a.severity or 0, reverse=True):
|
||||
@@ -238,3 +241,5 @@ class AlertManager:
|
||||
|
||||
html_parts.append("</body></html>")
|
||||
return "\n".join(html_parts)
|
||||
|
||||
|
||||
|
||||
@@ -5,13 +5,14 @@ Calculates timing advantage and suspicious activity scores.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, date, timedelta
|
||||
from datetime import date, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import and_, func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pote.db.models import MarketAlert, Security, Trade
|
||||
from pote.db.models import MarketAlert, Official, Security, Trade
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,7 +27,9 @@ class DisclosureCorrelator:
|
||||
"""Initialize disclosure correlator."""
|
||||
self.session = session
|
||||
|
||||
def get_alerts_before_trade(self, trade: Trade, lookback_days: int = 30) -> list[MarketAlert]:
|
||||
def get_alerts_before_trade(
|
||||
self, trade: Trade, lookback_days: int = 30
|
||||
) -> list[MarketAlert]:
|
||||
"""
|
||||
Get market alerts that occurred BEFORE a trade.
|
||||
|
||||
@@ -47,10 +50,14 @@ class DisclosureCorrelator:
|
||||
# Convert dates to datetime for comparison
|
||||
from datetime import datetime
|
||||
|
||||
start_dt = datetime.combine(start_date, datetime.min.time()).replace(tzinfo=UTC)
|
||||
end_dt = datetime.combine(end_date, datetime.max.time()).replace(tzinfo=UTC)
|
||||
start_dt = datetime.combine(start_date, datetime.min.time()).replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
end_dt = datetime.combine(end_date, datetime.max.time()).replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
|
||||
return (
|
||||
alerts = (
|
||||
self.session.query(MarketAlert)
|
||||
.filter(
|
||||
and_(
|
||||
@@ -63,6 +70,8 @@ class DisclosureCorrelator:
|
||||
.all()
|
||||
)
|
||||
|
||||
return alerts
|
||||
|
||||
def calculate_timing_score(
|
||||
self, trade: Trade, prior_alerts: list[MarketAlert]
|
||||
) -> dict[str, Any]:
|
||||
@@ -122,7 +131,8 @@ class DisclosureCorrelator:
|
||||
)
|
||||
elif suspicious:
|
||||
reason = (
|
||||
f"Trade occurred after {len(prior_alerts)} alerts. " f"Possible timing advantage."
|
||||
f"Trade occurred after {len(prior_alerts)} alerts. "
|
||||
f"Possible timing advantage."
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
@@ -160,27 +170,35 @@ class DisclosureCorrelator:
|
||||
timing_analysis = self.calculate_timing_score(trade, prior_alerts)
|
||||
|
||||
# Build full analysis
|
||||
return {
|
||||
analysis = {
|
||||
"trade_id": trade.id,
|
||||
"official_name": trade.official.name if trade.official else None,
|
||||
"ticker": trade.security.ticker if trade.security else None,
|
||||
"side": trade.side,
|
||||
"transaction_date": str(trade.transaction_date),
|
||||
"filing_date": str(trade.filing_date) if trade.filing_date else None,
|
||||
"value_range": f"${float(trade.value_min or 0):,.0f}"
|
||||
+ (f"-${float(trade.value_max):,.0f}" if trade.value_max else "+"),
|
||||
"value_range": f"${float(trade.value_min):,.0f}"
|
||||
+ (
|
||||
f"-${float(trade.value_max):,.0f}"
|
||||
if trade.value_max
|
||||
else "+"
|
||||
),
|
||||
**timing_analysis,
|
||||
"prior_alerts": [
|
||||
{
|
||||
"timestamp": str(alert.timestamp),
|
||||
"alert_type": alert.alert_type,
|
||||
"severity": alert.severity,
|
||||
"days_before_trade": (trade.transaction_date - alert.timestamp.date()).days,
|
||||
"days_before_trade": (
|
||||
trade.transaction_date - alert.timestamp.date()
|
||||
).days,
|
||||
}
|
||||
for alert in prior_alerts
|
||||
],
|
||||
}
|
||||
|
||||
return analysis
|
||||
|
||||
def analyze_recent_disclosures(
|
||||
self, days: int = 7, min_timing_score: float = 50
|
||||
) -> list[dict[str, Any]]:
|
||||
@@ -219,7 +237,9 @@ class DisclosureCorrelator:
|
||||
f"Found {len(suspicious_trades)} trades with timing score >= {min_timing_score}"
|
||||
)
|
||||
|
||||
return sorted(suspicious_trades, key=lambda x: x["timing_score"], reverse=True)
|
||||
return sorted(
|
||||
suspicious_trades, key=lambda x: x["timing_score"], reverse=True
|
||||
)
|
||||
|
||||
def get_official_timing_pattern(
|
||||
self, official_id: int, lookback_days: int = 365
|
||||
@@ -238,7 +258,9 @@ class DisclosureCorrelator:
|
||||
|
||||
trades = (
|
||||
self.session.query(Trade)
|
||||
.filter(and_(Trade.official_id == official_id, Trade.transaction_date >= since_date))
|
||||
.filter(
|
||||
and_(Trade.official_id == official_id, Trade.transaction_date >= since_date)
|
||||
)
|
||||
.join(Trade.security)
|
||||
.all()
|
||||
)
|
||||
@@ -263,7 +285,9 @@ class DisclosureCorrelator:
|
||||
highly_suspicious = sum(1 for a in analyses if a.get("highly_suspicious", False))
|
||||
|
||||
avg_timing_score = (
|
||||
sum(a["timing_score"] for a in analyses) / total_trades if total_trades > 0 else 0
|
||||
sum(a["timing_score"] for a in analyses) / total_trades
|
||||
if total_trades > 0
|
||||
else 0
|
||||
)
|
||||
|
||||
# Determine pattern
|
||||
@@ -287,7 +311,9 @@ class DisclosureCorrelator:
|
||||
"analyses": analyses,
|
||||
}
|
||||
|
||||
def get_ticker_timing_analysis(self, ticker: str, lookback_days: int = 365) -> dict[str, Any]:
|
||||
def get_ticker_timing_analysis(
|
||||
self, ticker: str, lookback_days: int = 365
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Analyze timing patterns for a specific ticker.
|
||||
|
||||
@@ -303,7 +329,9 @@ class DisclosureCorrelator:
|
||||
trades = (
|
||||
self.session.query(Trade)
|
||||
.join(Trade.security)
|
||||
.filter(and_(Security.ticker == ticker, Trade.transaction_date >= since_date))
|
||||
.filter(
|
||||
and_(Security.ticker == ticker, Trade.transaction_date >= since_date)
|
||||
)
|
||||
.join(Trade.official)
|
||||
.all()
|
||||
)
|
||||
@@ -322,6 +350,10 @@ class DisclosureCorrelator:
|
||||
"trade_count": len(analyses),
|
||||
"trades_with_alerts": sum(1 for a in analyses if a["alert_count"] > 0),
|
||||
"suspicious_count": sum(1 for a in analyses if a["suspicious"]),
|
||||
"avg_timing_score": round(sum(a["timing_score"] for a in analyses) / len(analyses), 2),
|
||||
"avg_timing_score": round(
|
||||
sum(a["timing_score"] for a in analyses) / len(analyses), 2
|
||||
),
|
||||
"analyses": sorted(analyses, key=lambda x: x["timing_score"], reverse=True),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ Detects unusual activity: volume spikes, price movements, volatility.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
@@ -59,7 +59,7 @@ class MarketMonitor:
|
||||
Returns:
|
||||
List of alerts detected
|
||||
"""
|
||||
alerts: list[dict[str, Any]] = []
|
||||
alerts = []
|
||||
|
||||
try:
|
||||
stock = yf.Ticker(ticker)
|
||||
@@ -90,7 +90,7 @@ class MarketMonitor:
|
||||
{
|
||||
"ticker": ticker,
|
||||
"alert_type": "unusual_volume",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {
|
||||
"current_volume": int(current_volume),
|
||||
"avg_volume": int(avg_volume),
|
||||
@@ -109,8 +109,10 @@ class MarketMonitor:
|
||||
alerts.append(
|
||||
{
|
||||
"ticker": ticker,
|
||||
"alert_type": "price_spike" if price_change > 0 else "price_drop",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"alert_type": "price_spike"
|
||||
if price_change > 0
|
||||
else "price_drop",
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {
|
||||
"current_price": float(current_price),
|
||||
"prev_price": float(prev["Close"]),
|
||||
@@ -127,12 +129,14 @@ class MarketMonitor:
|
||||
if len(hist) >= 5:
|
||||
recent_volatility = hist["Close"].iloc[-5:].pct_change().abs().mean()
|
||||
if recent_volatility > avg_price_change * 2 and avg_price_change > 0:
|
||||
severity = min(10, int((recent_volatility / avg_price_change) - 1))
|
||||
severity = min(
|
||||
10, int((recent_volatility / avg_price_change) - 1)
|
||||
)
|
||||
alerts.append(
|
||||
{
|
||||
"ticker": ticker,
|
||||
"alert_type": "high_volatility",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {
|
||||
"recent_volatility": round(recent_volatility * 100, 2),
|
||||
"avg_volatility": round(avg_price_change * 100, 2),
|
||||
@@ -223,7 +227,7 @@ class MarketMonitor:
|
||||
Returns:
|
||||
List of MarketAlert objects
|
||||
"""
|
||||
since = datetime.now(UTC) - timedelta(days=days)
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
|
||||
query = self.session.query(MarketAlert).filter(MarketAlert.timestamp >= since)
|
||||
|
||||
@@ -248,7 +252,7 @@ class MarketMonitor:
|
||||
Returns:
|
||||
Dict mapping ticker to alert summary
|
||||
"""
|
||||
since = datetime.now(UTC) - timedelta(days=days)
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
@@ -274,3 +278,5 @@ class MarketMonitor:
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
|
||||
@@ -5,12 +5,13 @@ Identifies recurring suspicious behavior and trading patterns.
|
||||
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import and_, func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pote.db.models import Official, Security, Trade
|
||||
from pote.db.models import MarketAlert, Official, Security, Trade
|
||||
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -59,7 +60,9 @@ class PatternDetector:
|
||||
.all()
|
||||
)
|
||||
|
||||
logger.info(f"Analyzing {len(officials_with_trades)} officials with {min_trades}+ trades")
|
||||
logger.info(
|
||||
f"Analyzing {len(officials_with_trades)} officials with {min_trades}+ trades"
|
||||
)
|
||||
|
||||
rankings = []
|
||||
|
||||
@@ -67,7 +70,9 @@ class PatternDetector:
|
||||
official_id, name, chamber, party, state, trade_count = official_data
|
||||
|
||||
# Get timing pattern
|
||||
pattern = self.correlator.get_official_timing_pattern(official_id, lookback_days)
|
||||
pattern = self.correlator.get_official_timing_pattern(
|
||||
official_id, lookback_days
|
||||
)
|
||||
|
||||
if pattern["trade_count"] == 0:
|
||||
continue
|
||||
@@ -123,7 +128,9 @@ class PatternDetector:
|
||||
rankings = self.rank_officials_by_timing(lookback_days, min_trades=5)
|
||||
|
||||
# Filter for high suspicious rates
|
||||
offenders = [r for r in rankings if r["suspicious_rate"] >= min_suspicious_rate * 100]
|
||||
offenders = [
|
||||
r for r in rankings if r["suspicious_rate"] >= min_suspicious_rate * 100
|
||||
]
|
||||
|
||||
logger.info(
|
||||
f"Found {len(offenders)} officials with {min_suspicious_rate*100}%+ suspicious trades"
|
||||
@@ -148,7 +155,9 @@ class PatternDetector:
|
||||
|
||||
# Get tickers with enough trades
|
||||
tickers_with_trades = (
|
||||
self.session.query(Security.ticker, func.count(Trade.id).label("trade_count"))
|
||||
self.session.query(
|
||||
Security.ticker, func.count(Trade.id).label("trade_count")
|
||||
)
|
||||
.join(Trade)
|
||||
.filter(Trade.transaction_date >= since_date)
|
||||
.group_by(Security.ticker)
|
||||
@@ -160,8 +169,10 @@ class PatternDetector:
|
||||
|
||||
ticker_patterns = []
|
||||
|
||||
for ticker, _trade_count in tickers_with_trades:
|
||||
analysis = self.correlator.get_ticker_timing_analysis(ticker, lookback_days)
|
||||
for ticker, trade_count in tickers_with_trades:
|
||||
analysis = self.correlator.get_ticker_timing_analysis(
|
||||
ticker, lookback_days
|
||||
)
|
||||
|
||||
if analysis["trade_count"] == 0:
|
||||
continue
|
||||
@@ -188,7 +199,9 @@ class PatternDetector:
|
||||
|
||||
return ticker_patterns
|
||||
|
||||
def get_sector_timing_analysis(self, lookback_days: int = 365) -> dict[str, dict[str, Any]]:
|
||||
def get_sector_timing_analysis(
|
||||
self, lookback_days: int = 365
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""
|
||||
Analyze timing patterns by sector.
|
||||
|
||||
@@ -239,7 +252,7 @@ class PatternDetector:
|
||||
sector_stats[sector]["suspicious_count"] += 1
|
||||
|
||||
# Calculate averages
|
||||
for stats in sector_stats.values():
|
||||
for sector, stats in sector_stats.items():
|
||||
if stats["trade_count"] > 0:
|
||||
stats["avg_timing_score"] = round(
|
||||
stats["total_timing_score"] / stats["trade_count"], 2
|
||||
@@ -253,7 +266,9 @@ class PatternDetector:
|
||||
|
||||
return sector_stats
|
||||
|
||||
def get_party_comparison(self, lookback_days: int = 365) -> dict[str, dict[str, Any]]:
|
||||
def get_party_comparison(
|
||||
self, lookback_days: int = 365
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""
|
||||
Compare timing patterns between political parties.
|
||||
|
||||
@@ -288,7 +303,7 @@ class PatternDetector:
|
||||
party_stats[party]["officials"].append(ranking)
|
||||
|
||||
# Calculate averages
|
||||
for stats in party_stats.values():
|
||||
for party, stats in party_stats.items():
|
||||
if stats["total_trades"] > 0:
|
||||
stats["avg_timing_score"] = round(
|
||||
stats["total_timing_score"] / stats["total_trades"], 2
|
||||
@@ -321,7 +336,7 @@ class PatternDetector:
|
||||
# Calculate summary statistics
|
||||
total_officials = len(official_rankings)
|
||||
total_offenders = len(repeat_offenders)
|
||||
|
||||
|
||||
avg_timing_score = (
|
||||
sum(r["avg_timing_score"] for r in official_rankings) / total_officials
|
||||
if total_officials > 0
|
||||
@@ -341,3 +356,5 @@ class PatternDetector:
|
||||
"sector_analysis": sector_analysis,
|
||||
"party_comparison": party_comparison,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -8,3 +8,5 @@ from .email_reporter import EmailReporter
|
||||
from .report_generator import ReportGenerator
|
||||
|
||||
__all__ = ["EmailReporter", "ReportGenerator"]
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import logging
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from typing import List, Optional
|
||||
|
||||
from pote.config import settings
|
||||
|
||||
@@ -19,30 +20,31 @@ class EmailReporter:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
smtp_host: str | None = None,
|
||||
smtp_port: int | None = None,
|
||||
smtp_user: str | None = None,
|
||||
smtp_password: str | None = None,
|
||||
from_email: str | None = None,
|
||||
smtp_host: Optional[str] = None,
|
||||
smtp_port: Optional[int] = None,
|
||||
smtp_user: Optional[str] = None,
|
||||
smtp_password: Optional[str] = None,
|
||||
from_email: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Initialize email reporter.
|
||||
|
||||
If parameters are not provided, will attempt to use settings from config.
|
||||
"""
|
||||
# Settings always defines these fields (with defaults), so direct access is safe.
|
||||
self.smtp_host: str = smtp_host or settings.smtp_host
|
||||
self.smtp_port: int = smtp_port or settings.smtp_port
|
||||
self.smtp_user: str = smtp_user or settings.smtp_user
|
||||
self.smtp_password: str = smtp_password or settings.smtp_password
|
||||
self.from_email: str = from_email or settings.from_email or "pote@localhost"
|
||||
self.smtp_host = smtp_host or getattr(settings, "smtp_host", "localhost")
|
||||
self.smtp_port = smtp_port or getattr(settings, "smtp_port", 587)
|
||||
self.smtp_user = smtp_user or getattr(settings, "smtp_user", None)
|
||||
self.smtp_password = smtp_password or getattr(settings, "smtp_password", None)
|
||||
self.from_email = from_email or getattr(
|
||||
settings, "from_email", "pote@localhost"
|
||||
)
|
||||
|
||||
def send_report(
|
||||
self,
|
||||
to_emails: list[str],
|
||||
to_emails: List[str],
|
||||
subject: str,
|
||||
body_text: str,
|
||||
body_html: str | None = None,
|
||||
body_html: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Send an email report.
|
||||
@@ -111,3 +113,4 @@ class EmailReporter:
|
||||
except Exception as e:
|
||||
logger.error(f"SMTP connection test failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ Generates formatted reports from database data.
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -27,15 +27,13 @@ class ReportGenerator:
|
||||
self.detector = PatternDetector(session)
|
||||
|
||||
def generate_daily_summary(
|
||||
self, report_date: date | None = None, *, lookback_days: int = 1
|
||||
) -> dict[str, Any]:
|
||||
self, report_date: Optional[date] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate a daily summary report.
|
||||
|
||||
Args:
|
||||
report_date: Date to generate report for (defaults to today)
|
||||
lookback_days: Include trades filed in the last N days ending on report_date
|
||||
(defaults to 1, meaning only filings on report_date).
|
||||
|
||||
Returns:
|
||||
Dictionary containing report data
|
||||
@@ -43,19 +41,12 @@ class ReportGenerator:
|
||||
if report_date is None:
|
||||
report_date = date.today()
|
||||
|
||||
if lookback_days < 1:
|
||||
raise ValueError("lookback_days must be >= 1")
|
||||
|
||||
start_of_day = datetime.combine(report_date, datetime.min.time())
|
||||
end_of_day = datetime.combine(report_date, datetime.max.time())
|
||||
|
||||
filing_start_date = report_date - timedelta(days=lookback_days - 1)
|
||||
|
||||
# Trades filed within the lookback window (inclusive)
|
||||
# Count new trades filed today
|
||||
new_trades = (
|
||||
self.session.query(Trade)
|
||||
.filter(Trade.filing_date >= filing_start_date, Trade.filing_date <= report_date)
|
||||
.all()
|
||||
self.session.query(Trade).filter(Trade.filing_date == report_date).all()
|
||||
)
|
||||
|
||||
# Count market alerts today
|
||||
@@ -69,7 +60,7 @@ class ReportGenerator:
|
||||
)
|
||||
|
||||
# Get high-severity alerts
|
||||
critical_alerts = [a for a in new_alerts if (a.severity or 0) >= 7]
|
||||
critical_alerts = [a for a in new_alerts if a.severity >= 7]
|
||||
|
||||
# Get suspicious timing matches
|
||||
suspicious_trades = []
|
||||
@@ -80,8 +71,6 @@ class ReportGenerator:
|
||||
|
||||
return {
|
||||
"date": report_date,
|
||||
"filing_start_date": filing_start_date,
|
||||
"lookback_days": lookback_days,
|
||||
"new_trades_count": len(new_trades),
|
||||
"new_trades": [
|
||||
{
|
||||
@@ -110,7 +99,7 @@ class ReportGenerator:
|
||||
"suspicious_trades": suspicious_trades,
|
||||
}
|
||||
|
||||
def generate_weekly_summary(self) -> dict[str, Any]:
|
||||
def generate_weekly_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate a weekly summary report.
|
||||
|
||||
@@ -121,7 +110,9 @@ class ReportGenerator:
|
||||
|
||||
# Most active officials
|
||||
active_officials = (
|
||||
self.session.query(Official.name, func.count(Trade.id).label("trade_count"))
|
||||
self.session.query(
|
||||
Official.name, func.count(Trade.id).label("trade_count")
|
||||
)
|
||||
.join(Trade)
|
||||
.filter(Trade.filing_date >= week_ago)
|
||||
.group_by(Official.id, Official.name)
|
||||
@@ -132,7 +123,9 @@ class ReportGenerator:
|
||||
|
||||
# Most traded securities
|
||||
active_securities = (
|
||||
self.session.query(Security.ticker, func.count(Trade.id).label("trade_count"))
|
||||
self.session.query(
|
||||
Security.ticker, func.count(Trade.id).label("trade_count")
|
||||
)
|
||||
.join(Trade)
|
||||
.filter(Trade.filing_date >= week_ago)
|
||||
.group_by(Security.id, Security.ticker)
|
||||
@@ -142,10 +135,8 @@ class ReportGenerator:
|
||||
)
|
||||
|
||||
# Get top suspicious patterns
|
||||
# Same fix as branch docs/deploy-email-closed: the old kwarg names never
|
||||
# existed on identify_repeat_offenders and failed every weekly run.
|
||||
repeat_offenders = self.detector.identify_repeat_offenders(
|
||||
lookback_days=7, min_suspicious_rate=0.4
|
||||
days_lookback=7, min_suspicious_trades=2, min_timing_score=40
|
||||
)
|
||||
|
||||
return {
|
||||
@@ -155,13 +146,14 @@ class ReportGenerator:
|
||||
{"name": name, "trade_count": count} for name, count in active_officials
|
||||
],
|
||||
"most_traded_securities": [
|
||||
{"ticker": ticker, "trade_count": count} for ticker, count in active_securities
|
||||
{"ticker": ticker, "trade_count": count}
|
||||
for ticker, count in active_securities
|
||||
],
|
||||
"repeat_offenders_count": len(repeat_offenders),
|
||||
"repeat_offenders": repeat_offenders[:5], # Top 5
|
||||
}
|
||||
|
||||
def format_as_text(self, report_data: dict[str, Any], report_type: str) -> str:
|
||||
def format_as_text(self, report_data: Dict[str, Any], report_type: str) -> str:
|
||||
"""
|
||||
Format report data as plain text.
|
||||
|
||||
@@ -174,26 +166,20 @@ class ReportGenerator:
|
||||
"""
|
||||
if report_type == "daily":
|
||||
return self._format_daily_text(report_data)
|
||||
if report_type == "weekly":
|
||||
elif report_type == "weekly":
|
||||
return self._format_weekly_text(report_data)
|
||||
return str(report_data)
|
||||
|
||||
def _format_daily_text(self, data: dict[str, Any]) -> str:
|
||||
"""Format daily report as plain text."""
|
||||
if data.get("lookback_days", 1) > 1:
|
||||
trades_label = (
|
||||
f" • Trades Filed (last {data['lookback_days']} days): {data['new_trades_count']}"
|
||||
)
|
||||
else:
|
||||
trades_label = f" • New Trades Filed: {data['new_trades_count']}"
|
||||
return str(report_data)
|
||||
|
||||
def _format_daily_text(self, data: Dict[str, Any]) -> str:
|
||||
"""Format daily report as plain text."""
|
||||
lines = [
|
||||
"=" * 70,
|
||||
f"POTE DAILY REPORT - {data['date']}",
|
||||
"=" * 70,
|
||||
"",
|
||||
"📊 SUMMARY",
|
||||
trades_label,
|
||||
f" • New Trades Filed: {data['new_trades_count']}",
|
||||
f" • Market Alerts: {data['market_alerts_count']}",
|
||||
f" • Critical Alerts (≥7 severity): {data['critical_alerts_count']}",
|
||||
f" • Suspicious Timing Trades: {data['suspicious_trades_count']}",
|
||||
@@ -241,7 +227,7 @@ class ReportGenerator:
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _format_weekly_text(self, data: dict[str, Any]) -> str:
|
||||
def _format_weekly_text(self, data: Dict[str, Any]) -> str:
|
||||
"""Format weekly report as plain text."""
|
||||
lines = [
|
||||
"=" * 70,
|
||||
@@ -260,7 +246,9 @@ class ReportGenerator:
|
||||
lines.append(f" • {security['ticker']}: {security['trade_count']} trades")
|
||||
|
||||
if data["repeat_offenders"]:
|
||||
lines.extend(["", f"⚠️ REPEAT OFFENDERS ({data['repeat_offenders_count']} total)"])
|
||||
lines.extend(
|
||||
["", f"⚠️ REPEAT OFFENDERS ({data['repeat_offenders_count']} total)"]
|
||||
)
|
||||
for offender in data["repeat_offenders"]:
|
||||
lines.append(
|
||||
f" • {offender['official_name']}: "
|
||||
@@ -279,7 +267,7 @@ class ReportGenerator:
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def format_as_html(self, report_data: dict[str, Any], report_type: str) -> str:
|
||||
def format_as_html(self, report_data: Dict[str, Any], report_type: str) -> str:
|
||||
"""
|
||||
Format report data as HTML.
|
||||
|
||||
@@ -292,17 +280,13 @@ class ReportGenerator:
|
||||
"""
|
||||
if report_type == "daily":
|
||||
return self._format_daily_html(report_data)
|
||||
if report_type == "weekly":
|
||||
elif report_type == "weekly":
|
||||
return self._format_weekly_html(report_data)
|
||||
return f"<pre>{report_data}</pre>"
|
||||
|
||||
def _format_daily_html(self, data: dict[str, Any]) -> str:
|
||||
"""Format daily report as HTML."""
|
||||
if data.get("lookback_days", 1) > 1:
|
||||
new_trades_label = f"Trades Filed (last {data['lookback_days']} days):"
|
||||
else:
|
||||
new_trades_label = "New Trades:"
|
||||
return f"<pre>{report_data}</pre>"
|
||||
|
||||
def _format_daily_html(self, data: Dict[str, Any]) -> str:
|
||||
"""Format daily report as HTML."""
|
||||
html = f"""
|
||||
<html>
|
||||
<head>
|
||||
@@ -320,10 +304,10 @@ class ReportGenerator:
|
||||
</head>
|
||||
<body>
|
||||
<h1>POTE Daily Report - {data['date']}</h1>
|
||||
|
||||
|
||||
<div class="summary">
|
||||
<h2>📊 Summary</h2>
|
||||
<div class="stat"><strong>{new_trades_label}</strong> {data['new_trades_count']}</div>
|
||||
<div class="stat"><strong>New Trades:</strong> {data['new_trades_count']}</div>
|
||||
<div class="stat"><strong>Market Alerts:</strong> {data['market_alerts_count']}</div>
|
||||
<div class="stat"><strong>Critical Alerts:</strong> {data['critical_alerts_count']}</div>
|
||||
<div class="stat"><strong>Suspicious Trades:</strong> {data['suspicious_trades_count']}</div>
|
||||
@@ -335,7 +319,7 @@ class ReportGenerator:
|
||||
for t in data["new_trades"][:10]:
|
||||
html += f"""
|
||||
<div class="trade">
|
||||
<strong>{t['official']}</strong>: {t['side']} {t['ticker']}
|
||||
<strong>{t['official']}</strong>: {t['side']} {t['ticker']}
|
||||
(${t['value_min']:,.0f} - ${t['value_max']:,.0f}) on {t['transaction_date']}
|
||||
</div>
|
||||
"""
|
||||
@@ -345,7 +329,7 @@ class ReportGenerator:
|
||||
for a in data["critical_alerts"][:5]:
|
||||
html += f"""
|
||||
<div class="alert critical">
|
||||
<strong>{a['ticker']}</strong>: {a['type']} (severity {a['severity']})
|
||||
<strong>{a['ticker']}</strong>: {a['type']} (severity {a['severity']})
|
||||
at {a['timestamp'].strftime('%H:%M:%S')}
|
||||
</div>
|
||||
"""
|
||||
@@ -370,7 +354,7 @@ class ReportGenerator:
|
||||
|
||||
return html
|
||||
|
||||
def _format_weekly_html(self, data: dict[str, Any]) -> str:
|
||||
def _format_weekly_html(self, data: Dict[str, Any]) -> str:
|
||||
"""Format weekly report as HTML."""
|
||||
html = f"""
|
||||
<html>
|
||||
@@ -388,7 +372,7 @@ class ReportGenerator:
|
||||
<body>
|
||||
<h1>POTE Weekly Report</h1>
|
||||
<p><strong>Period:</strong> {data['period_start']} to {data['period_end']}</p>
|
||||
|
||||
|
||||
<h2>👥 Most Active Officials</h2>
|
||||
<table>
|
||||
<tr><th>Official</th><th>Trade Count</th></tr>
|
||||
@@ -399,7 +383,7 @@ class ReportGenerator:
|
||||
|
||||
html += """
|
||||
</table>
|
||||
|
||||
|
||||
<h2>📈 Most Traded Securities</h2>
|
||||
<table>
|
||||
<tr><th>Ticker</th><th>Trade Count</th></tr>
|
||||
@@ -436,3 +420,4 @@ class ReportGenerator:
|
||||
"""
|
||||
|
||||
return html
|
||||
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ def test_db_session() -> Session:
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
session_factory = sessionmaker(bind=engine)
|
||||
session = session_factory()
|
||||
TestSessionLocal = sessionmaker(bind=engine)
|
||||
session = TestSessionLocal()
|
||||
|
||||
yield session
|
||||
|
||||
|
||||
+17
-21
@@ -1,28 +1,27 @@
|
||||
"""Tests for analytics module."""
|
||||
|
||||
import pytest
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from pote.analytics.returns import ReturnCalculator
|
||||
from pote.analytics.benchmarks import BenchmarkComparison
|
||||
from pote.analytics.metrics import PerformanceMetrics
|
||||
from pote.analytics.returns import ReturnCalculator
|
||||
from pote.db.models import Price, Security, Trade
|
||||
from pote.db.models import Official, Security, Trade, Price
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_prices(test_db_session, sample_security):
|
||||
"""Create sample price data for testing."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
# Add SPY (benchmark) prices
|
||||
spy = Security(ticker="SPY", name="SPDR S&P 500 ETF")
|
||||
session.add(spy)
|
||||
session.flush()
|
||||
|
||||
|
||||
base_date = date(2024, 1, 1)
|
||||
|
||||
|
||||
# Create SPY prices
|
||||
for i in range(100):
|
||||
price = Price(
|
||||
@@ -35,7 +34,7 @@ def sample_prices(test_db_session, sample_security):
|
||||
volume=1000000,
|
||||
)
|
||||
session.add(price)
|
||||
|
||||
|
||||
# Create prices for sample_security (AAPL)
|
||||
for i in range(100):
|
||||
price = Price(
|
||||
@@ -48,7 +47,7 @@ def sample_prices(test_db_session, sample_security):
|
||||
volume=50000000,
|
||||
)
|
||||
session.add(price)
|
||||
|
||||
|
||||
session.commit()
|
||||
return session
|
||||
|
||||
@@ -81,9 +80,7 @@ def test_return_calculator_basic(test_db_session, sample_official, sample_securi
|
||||
assert "exit_price" in result
|
||||
|
||||
|
||||
def test_return_calculator_sell_trade(
|
||||
test_db_session, sample_official, sample_security, sample_prices
|
||||
):
|
||||
def test_return_calculator_sell_trade(test_db_session, sample_official, sample_security, sample_prices):
|
||||
session = test_db_session
|
||||
"""Test return calculation for sell trade."""
|
||||
trade = Trade(
|
||||
@@ -133,7 +130,7 @@ def test_benchmark_comparison(test_db_session, sample_official, sample_security,
|
||||
"""Test benchmark comparison."""
|
||||
# Create trade and SPY security
|
||||
spy = session.query(Security).filter_by(ticker="SPY").first()
|
||||
|
||||
|
||||
trade = Trade(
|
||||
official_id=sample_official.id,
|
||||
security_id=spy.id,
|
||||
@@ -157,14 +154,12 @@ def test_benchmark_comparison(test_db_session, sample_official, sample_security,
|
||||
assert "beat_market" in result
|
||||
|
||||
|
||||
def test_performance_metrics_official(
|
||||
test_db_session, sample_official, sample_security, sample_prices
|
||||
):
|
||||
def test_performance_metrics_official(test_db_session, sample_official, sample_security, sample_prices):
|
||||
session = test_db_session
|
||||
"""Test official performance metrics."""
|
||||
# Create multiple trades
|
||||
spy = session.query(Security).filter_by(ticker="SPY").first()
|
||||
|
||||
|
||||
for i in range(3):
|
||||
trade = Trade(
|
||||
official_id=sample_official.id,
|
||||
@@ -176,7 +171,7 @@ def test_performance_metrics_official(
|
||||
value_max=Decimal("50000"),
|
||||
)
|
||||
session.add(trade)
|
||||
|
||||
|
||||
session.commit()
|
||||
|
||||
# Get performance metrics
|
||||
@@ -192,7 +187,7 @@ def test_multiple_windows(test_db_session, sample_official, sample_security, sam
|
||||
session = test_db_session
|
||||
"""Test calculating returns for multiple windows."""
|
||||
spy = session.query(Security).filter_by(ticker="SPY").first()
|
||||
|
||||
|
||||
trade = Trade(
|
||||
official_id=sample_official.id,
|
||||
security_id=spy.id,
|
||||
@@ -236,7 +231,7 @@ def test_sector_analysis(test_db_session, sample_official, sample_prices):
|
||||
value_max=Decimal("50000"),
|
||||
)
|
||||
session.add(trade)
|
||||
|
||||
|
||||
session.commit()
|
||||
|
||||
metrics = PerformanceMetrics(session)
|
||||
@@ -262,7 +257,7 @@ def test_timing_analysis(test_db_session, sample_official, sample_security):
|
||||
value_max=Decimal("50000"),
|
||||
)
|
||||
session.add(trade)
|
||||
|
||||
|
||||
session.commit()
|
||||
|
||||
metrics = PerformanceMetrics(session)
|
||||
@@ -270,3 +265,4 @@ def test_timing_analysis(test_db_session, sample_official, sample_security):
|
||||
|
||||
assert "avg_disclosure_lag_days" in timing
|
||||
assert timing["avg_disclosure_lag_days"] > 0
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
"""Integration tests for analytics with real-ish data."""
|
||||
|
||||
import pytest
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from pote.analytics.returns import ReturnCalculator
|
||||
from pote.analytics.benchmarks import BenchmarkComparison
|
||||
from pote.analytics.metrics import PerformanceMetrics
|
||||
from pote.analytics.returns import ReturnCalculator
|
||||
from pote.db.models import Official, Price, Security, Trade
|
||||
from pote.db.models import Official, Security, Trade, Price
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -40,13 +39,13 @@ def full_test_data(test_db_session):
|
||||
# Create price data for NVDA (upward trend)
|
||||
base_date = date(2024, 1, 1)
|
||||
nvda_base_price = Decimal("495.00")
|
||||
|
||||
|
||||
for i in range(120):
|
||||
current_date = base_date + timedelta(days=i)
|
||||
# Simulate upward trend: +0.5% per day on average
|
||||
price_change = Decimal(i) * Decimal("2.50") # ~50% gain over 120 days
|
||||
current_price = nvda_base_price + price_change
|
||||
|
||||
|
||||
price = Price(
|
||||
security_id=nvda.id,
|
||||
date=current_date,
|
||||
@@ -60,12 +59,12 @@ def full_test_data(test_db_session):
|
||||
|
||||
# Create price data for SPY (slower upward trend - ~10% over 120 days)
|
||||
spy_base_price = Decimal("450.00")
|
||||
|
||||
|
||||
for i in range(120):
|
||||
current_date = base_date + timedelta(days=i)
|
||||
price_change = Decimal(i) * Decimal("0.35")
|
||||
current_price = spy_base_price + price_change
|
||||
|
||||
|
||||
price = Price(
|
||||
security_id=spy.id,
|
||||
date=current_date,
|
||||
@@ -89,7 +88,7 @@ def full_test_data(test_db_session):
|
||||
value_min=Decimal("15001"),
|
||||
value_max=Decimal("50000"),
|
||||
)
|
||||
|
||||
|
||||
# Tuberville buys NVDA later (still good but less alpha)
|
||||
trade2 = Trade(
|
||||
official_id=tuberville.id,
|
||||
@@ -116,24 +115,22 @@ def test_return_calculation_with_real_data(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test return calculation with realistic price data."""
|
||||
calculator = ReturnCalculator(session)
|
||||
|
||||
|
||||
# Get Pelosi's NVDA trade
|
||||
trade = full_test_data["trades"][0]
|
||||
|
||||
|
||||
# Calculate 90-day return
|
||||
result = calculator.calculate_trade_return(trade, window_days=90)
|
||||
|
||||
|
||||
assert result is not None, "Should calculate return with available data"
|
||||
assert result["ticker"] == "NVDA"
|
||||
assert result["window_days"] == 90
|
||||
assert result["return_pct"] > 0, "NVDA should have positive return"
|
||||
|
||||
|
||||
# Entry around day 15, exit around day 105
|
||||
# Expected return: (720 - 532.5) / 532.5 = ~35%
|
||||
assert (
|
||||
30 < float(result["return_pct"]) < 50
|
||||
), f"Expected ~35% return, got {result['return_pct']}"
|
||||
|
||||
assert 30 < float(result["return_pct"]) < 50, f"Expected ~35% return, got {result['return_pct']}"
|
||||
|
||||
print(f"\n✅ NVDA 90-day return: {result['return_pct']:.2f}%")
|
||||
print(f" Entry: ${result['entry_price']} on {result['transaction_date']}")
|
||||
print(f" Exit: ${result['exit_price']} on {result['exit_date']}")
|
||||
@@ -143,22 +140,22 @@ def test_benchmark_comparison_with_real_data(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test benchmark comparison with SPY."""
|
||||
benchmark = BenchmarkComparison(session)
|
||||
|
||||
|
||||
# Get Pelosi's trade
|
||||
trade = full_test_data["trades"][0]
|
||||
|
||||
|
||||
# Compare to SPY
|
||||
result = benchmark.compare_trade_to_benchmark(trade, window_days=90, benchmark="SPY")
|
||||
|
||||
|
||||
assert result is not None
|
||||
assert result["ticker"] == "NVDA"
|
||||
assert result["benchmark"] == "SPY"
|
||||
|
||||
|
||||
# NVDA should beat SPY significantly
|
||||
assert result["beat_market"] is True
|
||||
assert float(result["abnormal_return"]) > 10, "NVDA should have strong alpha vs SPY"
|
||||
|
||||
print("\n✅ Benchmark Comparison:")
|
||||
|
||||
print(f"\n✅ Benchmark Comparison:")
|
||||
print(f" NVDA Return: {result['trade_return']:.2f}%")
|
||||
print(f" SPY Return: {result['benchmark_return']:.2f}%")
|
||||
print(f" Alpha: {result['abnormal_return']:+.2f}%")
|
||||
@@ -168,21 +165,21 @@ def test_official_performance_summary(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test official performance aggregation."""
|
||||
metrics = PerformanceMetrics(session)
|
||||
|
||||
|
||||
pelosi = full_test_data["officials"][0]
|
||||
|
||||
|
||||
# Get performance summary
|
||||
perf = metrics.official_performance(pelosi.id, window_days=90)
|
||||
|
||||
|
||||
assert perf["name"] == "Nancy Pelosi"
|
||||
assert perf["total_trades"] >= 1
|
||||
|
||||
|
||||
if perf.get("trades_analyzed", 0) > 0:
|
||||
assert "avg_return" in perf
|
||||
assert "avg_alpha" in perf
|
||||
assert "win_rate" in perf
|
||||
assert perf["win_rate"] >= 0 and perf["win_rate"] <= 1
|
||||
|
||||
|
||||
print(f"\n✅ {perf['name']} Performance:")
|
||||
print(f" Total Trades: {perf['total_trades']}")
|
||||
print(f" Average Return: {perf['avg_return']:.2f}%")
|
||||
@@ -194,17 +191,17 @@ def test_multiple_windows(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test calculating multiple time windows."""
|
||||
calculator = ReturnCalculator(session)
|
||||
|
||||
|
||||
trade = full_test_data["trades"][0]
|
||||
|
||||
|
||||
# Calculate for 30, 60, 90 days
|
||||
results = calculator.calculate_multiple_windows(trade, windows=[30, 60, 90])
|
||||
|
||||
|
||||
assert len(results) == 3, "Should calculate all three windows"
|
||||
|
||||
|
||||
# Returns should generally increase with longer windows (given upward trend)
|
||||
if 30 in results and 90 in results:
|
||||
print("\n✅ Multiple Windows:")
|
||||
print(f"\n✅ Multiple Windows:")
|
||||
for window in [30, 60, 90]:
|
||||
if window in results:
|
||||
print(f" {window:3d} days: {results[window]['return_pct']:+7.2f}%")
|
||||
@@ -214,13 +211,13 @@ def test_top_performers(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test top performer ranking."""
|
||||
metrics = PerformanceMetrics(session)
|
||||
|
||||
|
||||
top = metrics.top_performers(window_days=90, limit=5)
|
||||
|
||||
|
||||
assert isinstance(top, list)
|
||||
assert len(top) > 0
|
||||
|
||||
print("\n✅ Top Performers:")
|
||||
|
||||
print(f"\n✅ Top Performers:")
|
||||
for i, perf in enumerate(top, 1):
|
||||
if perf.get("trades_analyzed", 0) > 0:
|
||||
print(f" {i}. {perf['name']:20s} | Alpha: {perf['avg_alpha']:+6.2f}%")
|
||||
@@ -230,18 +227,18 @@ def test_system_statistics(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test system-wide statistics."""
|
||||
metrics = PerformanceMetrics(session)
|
||||
|
||||
|
||||
stats = metrics.summary_statistics(window_days=90)
|
||||
|
||||
|
||||
assert stats["total_officials"] >= 2
|
||||
assert stats["total_trades"] >= 2
|
||||
assert stats["total_securities"] >= 2
|
||||
|
||||
print("\n✅ System Statistics:")
|
||||
|
||||
print(f"\n✅ System Statistics:")
|
||||
print(f" Officials: {stats['total_officials']}")
|
||||
print(f" Trades: {stats['total_trades']}")
|
||||
print(f" Securities: {stats['total_securities']}")
|
||||
|
||||
|
||||
if stats.get("avg_alpha") is not None:
|
||||
print(f" Avg Alpha: {stats['avg_alpha']:+.2f}%")
|
||||
print(f" Beat Market: {stats['beat_market_rate']:.1%}")
|
||||
@@ -251,13 +248,13 @@ def test_disclosure_timing(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test disclosure lag analysis."""
|
||||
metrics = PerformanceMetrics(session)
|
||||
|
||||
|
||||
timing = metrics.timing_analysis()
|
||||
|
||||
|
||||
assert "avg_disclosure_lag_days" in timing
|
||||
assert timing["avg_disclosure_lag_days"] > 0
|
||||
|
||||
print("\n✅ Disclosure Timing:")
|
||||
|
||||
print(f"\n✅ Disclosure Timing:")
|
||||
print(f" Average Lag: {timing['avg_disclosure_lag_days']:.1f} days")
|
||||
print(f" Median Lag: {timing['median_disclosure_lag_days']} days")
|
||||
|
||||
@@ -266,27 +263,25 @@ def test_sector_analysis(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test sector-level analysis."""
|
||||
metrics = PerformanceMetrics(session)
|
||||
|
||||
|
||||
sectors = metrics.sector_analysis(window_days=90)
|
||||
|
||||
|
||||
assert isinstance(sectors, list)
|
||||
|
||||
|
||||
if sectors:
|
||||
print("\n✅ Sector Analysis:")
|
||||
print(f"\n✅ Sector Analysis:")
|
||||
for s in sectors:
|
||||
print(
|
||||
f" {s['sector']:15s} | {s['trade_count']} trades | Alpha: {s['avg_alpha']:+6.2f}%"
|
||||
)
|
||||
print(f" {s['sector']:15s} | {s['trade_count']} trades | Alpha: {s['avg_alpha']:+6.2f}%")
|
||||
|
||||
|
||||
def test_edge_case_missing_exit_price(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test handling of trade with no exit price available."""
|
||||
calculator = ReturnCalculator(session)
|
||||
|
||||
|
||||
nvda = session.query(Security).filter_by(ticker="NVDA").first()
|
||||
pelosi = session.query(Official).filter_by(name="Nancy Pelosi").first()
|
||||
|
||||
|
||||
# Create trade with transaction date far in future (no exit price)
|
||||
future_trade = Trade(
|
||||
official_id=pelosi.id,
|
||||
@@ -299,9 +294,9 @@ def test_edge_case_missing_exit_price(test_db_session, full_test_data):
|
||||
)
|
||||
session.add(future_trade)
|
||||
session.commit()
|
||||
|
||||
|
||||
result = calculator.calculate_trade_return(future_trade, window_days=90)
|
||||
|
||||
|
||||
assert result is None, "Should return None when price data unavailable"
|
||||
print("\n✅ Correctly handles missing price data")
|
||||
|
||||
@@ -310,10 +305,10 @@ def test_sell_trade_logic(test_db_session, full_test_data):
|
||||
session = test_db_session
|
||||
"""Test that sell trades have inverted return logic."""
|
||||
calculator = ReturnCalculator(session)
|
||||
|
||||
|
||||
nvda = session.query(Security).filter_by(ticker="NVDA").first()
|
||||
pelosi = session.query(Official).filter_by(name="Nancy Pelosi").first()
|
||||
|
||||
|
||||
# Create sell trade during uptrend (should show negative return)
|
||||
sell_trade = Trade(
|
||||
official_id=pelosi.id,
|
||||
@@ -326,10 +321,11 @@ def test_sell_trade_logic(test_db_session, full_test_data):
|
||||
)
|
||||
session.add(sell_trade)
|
||||
session.commit()
|
||||
|
||||
|
||||
result = calculator.calculate_trade_return(sell_trade, window_days=90)
|
||||
|
||||
|
||||
if result:
|
||||
# Selling during uptrend = negative return
|
||||
assert result["return_pct"] < 0, "Sell during uptrend should show negative return"
|
||||
print(f"\n✅ Sell trade return correctly inverted: {result['return_pct']:.2f}%")
|
||||
|
||||
|
||||
@@ -1,25 +1,24 @@
|
||||
"""Tests for disclosure correlation module."""
|
||||
|
||||
from datetime import UTC, date, datetime
|
||||
import pytest
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from pote.db.models import MarketAlert, Official, Security, Trade
|
||||
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
|
||||
from pote.db.models import Official, Security, Trade, MarketAlert
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def trade_with_alerts(test_db_session):
|
||||
"""Create a trade with prior market alerts."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
# Create official and security
|
||||
pelosi = Official(name="Nancy Pelosi", chamber="House", party="Democrat", state="CA")
|
||||
nvda = Security(ticker="NVDA", name="NVIDIA Corporation", sector="Technology")
|
||||
session.add_all([pelosi, nvda])
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create trade on Jan 15
|
||||
trade = Trade(
|
||||
official_id=pelosi.id,
|
||||
@@ -33,13 +32,13 @@ def trade_with_alerts(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create alerts BEFORE trade (suspicious)
|
||||
alerts = [
|
||||
MarketAlert(
|
||||
ticker="NVDA",
|
||||
alert_type="unusual_volume",
|
||||
timestamp=datetime(2024, 1, 10, 10, 30, tzinfo=UTC), # 5 days before
|
||||
timestamp=datetime(2024, 1, 10, 10, 30, tzinfo=timezone.utc), # 5 days before
|
||||
details={"multiplier": 3.5},
|
||||
price=Decimal("490.00"),
|
||||
volume=100000000,
|
||||
@@ -49,7 +48,7 @@ def trade_with_alerts(test_db_session):
|
||||
MarketAlert(
|
||||
ticker="NVDA",
|
||||
alert_type="price_spike",
|
||||
timestamp=datetime(2024, 1, 12, 14, 15, tzinfo=UTC), # 3 days before
|
||||
timestamp=datetime(2024, 1, 12, 14, 15, tzinfo=timezone.utc), # 3 days before
|
||||
details={"change_pct": 5.5},
|
||||
price=Decimal("505.00"),
|
||||
volume=85000000,
|
||||
@@ -59,7 +58,7 @@ def trade_with_alerts(test_db_session):
|
||||
MarketAlert(
|
||||
ticker="NVDA",
|
||||
alert_type="high_volatility",
|
||||
timestamp=datetime(2024, 1, 14, 16, 20, tzinfo=UTC), # 1 day before
|
||||
timestamp=datetime(2024, 1, 14, 16, 20, tzinfo=timezone.utc), # 1 day before
|
||||
details={"multiplier": 2.5},
|
||||
price=Decimal("510.00"),
|
||||
volume=90000000,
|
||||
@@ -69,7 +68,7 @@ def trade_with_alerts(test_db_session):
|
||||
]
|
||||
session.add_all(alerts)
|
||||
session.commit()
|
||||
|
||||
|
||||
return {
|
||||
"trade": trade,
|
||||
"official": pelosi,
|
||||
@@ -82,12 +81,12 @@ def trade_with_alerts(test_db_session):
|
||||
def trade_without_alerts(test_db_session):
|
||||
"""Create a trade without prior alerts (clean)."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
official = Official(name="John Smith", chamber="House", party="Republican", state="TX")
|
||||
security = Security(ticker="MSFT", name="Microsoft Corporation", sector="Technology")
|
||||
session.add_all([official, security])
|
||||
session.flush()
|
||||
|
||||
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
security_id=security.id,
|
||||
@@ -99,7 +98,7 @@ def trade_without_alerts(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.commit()
|
||||
|
||||
|
||||
return {
|
||||
"trade": trade,
|
||||
"official": official,
|
||||
@@ -111,12 +110,12 @@ def test_get_alerts_before_trade(test_db_session, trade_with_alerts):
|
||||
"""Test retrieving alerts before a trade."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
trade = trade_with_alerts["trade"]
|
||||
|
||||
|
||||
# Get alerts before trade
|
||||
prior_alerts = correlator.get_alerts_before_trade(trade, lookback_days=30)
|
||||
|
||||
|
||||
assert len(prior_alerts) == 3
|
||||
assert all(alert.ticker == "NVDA" for alert in prior_alerts)
|
||||
assert all(alert.timestamp.date() < trade.transaction_date for alert in prior_alerts)
|
||||
@@ -126,11 +125,11 @@ def test_get_alerts_before_trade_no_alerts(test_db_session, trade_without_alerts
|
||||
"""Test retrieving alerts when none exist."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
trade = trade_without_alerts["trade"]
|
||||
|
||||
|
||||
prior_alerts = correlator.get_alerts_before_trade(trade, lookback_days=30)
|
||||
|
||||
|
||||
assert len(prior_alerts) == 0
|
||||
|
||||
|
||||
@@ -138,12 +137,12 @@ def test_calculate_timing_score_high_suspicion(test_db_session, trade_with_alert
|
||||
"""Test timing score calculation for suspicious trade."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
trade = trade_with_alerts["trade"]
|
||||
alerts = trade_with_alerts["alerts"]
|
||||
|
||||
|
||||
timing_analysis = correlator.calculate_timing_score(trade, alerts)
|
||||
|
||||
|
||||
assert timing_analysis["timing_score"] > 60, "Should be suspicious with 3 alerts"
|
||||
assert timing_analysis["suspicious"] is True
|
||||
assert timing_analysis["alert_count"] == 3
|
||||
@@ -156,13 +155,13 @@ def test_calculate_timing_score_no_alerts(test_db_session):
|
||||
"""Test timing score with no prior alerts."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Create minimal trade
|
||||
official = Official(name="Test", chamber="House", party="Democrat", state="CA")
|
||||
security = Security(ticker="TEST", name="Test Corp")
|
||||
session.add_all([official, security])
|
||||
session.flush()
|
||||
|
||||
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
security_id=security.id,
|
||||
@@ -173,9 +172,9 @@ def test_calculate_timing_score_no_alerts(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.commit()
|
||||
|
||||
|
||||
timing_analysis = correlator.calculate_timing_score(trade, [])
|
||||
|
||||
|
||||
assert timing_analysis["timing_score"] == 0
|
||||
assert timing_analysis["suspicious"] is False
|
||||
assert timing_analysis["alert_count"] == 0
|
||||
@@ -185,13 +184,13 @@ def test_calculate_timing_score_factors(test_db_session):
|
||||
"""Test that timing score considers all factors correctly."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Create trade
|
||||
official = Official(name="Test", chamber="House", party="Democrat", state="CA")
|
||||
security = Security(ticker="TEST", name="Test Corp")
|
||||
session.add_all([official, security])
|
||||
session.flush()
|
||||
|
||||
|
||||
trade_date = date(2024, 1, 15)
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
@@ -203,47 +202,47 @@ def test_calculate_timing_score_factors(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Test with low severity alerts (should have lower score)
|
||||
low_sev_alerts = [
|
||||
MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=timezone.utc),
|
||||
severity=3,
|
||||
),
|
||||
MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 11, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2024, 1, 11, 12, 0, tzinfo=timezone.utc),
|
||||
severity=4,
|
||||
),
|
||||
]
|
||||
session.add_all(low_sev_alerts)
|
||||
session.commit()
|
||||
|
||||
|
||||
low_score = correlator.calculate_timing_score(trade, low_sev_alerts)
|
||||
|
||||
|
||||
# Test with high severity alerts (should have higher score)
|
||||
high_sev_alerts = [
|
||||
MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 13, 12, 0, tzinfo=UTC), # Recent
|
||||
timestamp=datetime(2024, 1, 13, 12, 0, tzinfo=timezone.utc), # Recent
|
||||
severity=9,
|
||||
),
|
||||
MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 14, 12, 0, tzinfo=UTC), # Very recent
|
||||
timestamp=datetime(2024, 1, 14, 12, 0, tzinfo=timezone.utc), # Very recent
|
||||
severity=8,
|
||||
),
|
||||
]
|
||||
session.add_all(high_sev_alerts)
|
||||
session.commit()
|
||||
|
||||
|
||||
high_score = correlator.calculate_timing_score(trade, high_sev_alerts)
|
||||
|
||||
|
||||
# High severity + recent should score higher
|
||||
assert high_score["timing_score"] > low_score["timing_score"]
|
||||
assert high_score["recent_alert_count"] > 0
|
||||
@@ -254,11 +253,11 @@ def test_analyze_trade_full(test_db_session, trade_with_alerts):
|
||||
"""Test complete trade analysis."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
trade = trade_with_alerts["trade"]
|
||||
|
||||
|
||||
analysis = correlator.analyze_trade(trade)
|
||||
|
||||
|
||||
# Check all required fields
|
||||
assert analysis["trade_id"] == trade.id
|
||||
assert analysis["official_name"] == "Nancy Pelosi"
|
||||
@@ -268,7 +267,7 @@ def test_analyze_trade_full(test_db_session, trade_with_alerts):
|
||||
assert analysis["timing_score"] > 0
|
||||
assert "prior_alerts" in analysis
|
||||
assert len(analysis["prior_alerts"]) == 3
|
||||
|
||||
|
||||
# Check alert details
|
||||
for alert_detail in analysis["prior_alerts"]:
|
||||
assert "timestamp" in alert_detail
|
||||
@@ -282,15 +281,16 @@ def test_analyze_recent_disclosures(test_db_session, trade_with_alerts, trade_wi
|
||||
"""Test batch analysis of recent disclosures."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Both trades were created "recently" (in fixture setup)
|
||||
suspicious_trades = correlator.analyze_recent_disclosures(
|
||||
days=365, min_timing_score=50 # Wide window to catch test data
|
||||
days=365, # Wide window to catch test data
|
||||
min_timing_score=50
|
||||
)
|
||||
|
||||
|
||||
# Should find at least the suspicious trade
|
||||
assert len(suspicious_trades) >= 1
|
||||
|
||||
|
||||
# Check sorting (highest score first)
|
||||
if len(suspicious_trades) > 1:
|
||||
for i in range(len(suspicious_trades) - 1):
|
||||
@@ -301,12 +301,12 @@ def test_get_official_timing_pattern(test_db_session, trade_with_alerts):
|
||||
"""Test official timing pattern analysis."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
official = trade_with_alerts["official"]
|
||||
|
||||
|
||||
# Use wide lookback to catch test data (trade is 2024-01-15)
|
||||
pattern = correlator.get_official_timing_pattern(official.id, lookback_days=3650)
|
||||
|
||||
|
||||
assert pattern["official_id"] == official.id
|
||||
assert pattern["trade_count"] >= 1
|
||||
assert pattern["trades_with_prior_alerts"] >= 1
|
||||
@@ -319,13 +319,13 @@ def test_get_official_timing_pattern_no_trades(test_db_session):
|
||||
"""Test official with no trades."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
official = Official(name="No Trades", chamber="House", party="Democrat", state="CA")
|
||||
session.add(official)
|
||||
session.commit()
|
||||
|
||||
|
||||
pattern = correlator.get_official_timing_pattern(official.id)
|
||||
|
||||
|
||||
assert pattern["trade_count"] == 0
|
||||
assert "No trades" in pattern["pattern"]
|
||||
|
||||
@@ -334,10 +334,10 @@ def test_get_ticker_timing_analysis(test_db_session, trade_with_alerts):
|
||||
"""Test ticker timing analysis."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Use wide lookback to catch test data
|
||||
analysis = correlator.get_ticker_timing_analysis("NVDA", lookback_days=3650)
|
||||
|
||||
|
||||
assert analysis["ticker"] == "NVDA"
|
||||
assert analysis["trade_count"] >= 1
|
||||
assert analysis["trades_with_alerts"] >= 1
|
||||
@@ -349,9 +349,9 @@ def test_get_ticker_timing_analysis_no_trades(test_db_session):
|
||||
"""Test ticker with no trades."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
analysis = correlator.get_ticker_timing_analysis("ZZZZ")
|
||||
|
||||
|
||||
assert analysis["ticker"] == "ZZZZ"
|
||||
assert analysis["trade_count"] == 0
|
||||
assert "No trades" in analysis["pattern"]
|
||||
@@ -361,13 +361,13 @@ def test_alerts_outside_lookback_window(test_db_session):
|
||||
"""Test that alerts outside lookback window are excluded."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Create trade and alerts
|
||||
official = Official(name="Test", chamber="House", party="Democrat", state="CA")
|
||||
security = Security(ticker="TEST", name="Test Corp")
|
||||
session.add_all([official, security])
|
||||
session.flush()
|
||||
|
||||
|
||||
trade_date = date(2024, 1, 15)
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
@@ -379,29 +379,29 @@ def test_alerts_outside_lookback_window(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Alert 2 days before (within window)
|
||||
recent_alert = MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 13, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2024, 1, 13, 12, 0, tzinfo=timezone.utc),
|
||||
severity=7,
|
||||
)
|
||||
|
||||
|
||||
# Alert 40 days before (outside 30-day window)
|
||||
old_alert = MarketAlert(
|
||||
ticker="TEST",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2023, 12, 6, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2023, 12, 6, 12, 0, tzinfo=timezone.utc),
|
||||
severity=8,
|
||||
)
|
||||
|
||||
|
||||
session.add_all([recent_alert, old_alert])
|
||||
session.commit()
|
||||
|
||||
|
||||
# Should only get recent alert
|
||||
alerts = correlator.get_alerts_before_trade(trade, lookback_days=30)
|
||||
|
||||
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0].timestamp.date() == date(2024, 1, 13)
|
||||
|
||||
@@ -410,14 +410,14 @@ def test_different_ticker_alerts_excluded(test_db_session):
|
||||
"""Test that alerts for different tickers are excluded."""
|
||||
session = test_db_session
|
||||
correlator = DisclosureCorrelator(session)
|
||||
|
||||
|
||||
# Create trade for NVDA
|
||||
official = Official(name="Test", chamber="House", party="Democrat", state="CA")
|
||||
nvda = Security(ticker="NVDA", name="NVIDIA")
|
||||
msft = Security(ticker="MSFT", name="Microsoft")
|
||||
session.add_all([official, nvda, msft])
|
||||
session.flush()
|
||||
|
||||
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
security_id=nvda.id,
|
||||
@@ -428,27 +428,28 @@ def test_different_ticker_alerts_excluded(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create alerts for both tickers
|
||||
nvda_alert = MarketAlert(
|
||||
ticker="NVDA",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=timezone.utc),
|
||||
severity=7,
|
||||
)
|
||||
|
||||
|
||||
msft_alert = MarketAlert(
|
||||
ticker="MSFT",
|
||||
alert_type="test",
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=UTC),
|
||||
timestamp=datetime(2024, 1, 10, 12, 0, tzinfo=timezone.utc),
|
||||
severity=8,
|
||||
)
|
||||
|
||||
|
||||
session.add_all([nvda_alert, msft_alert])
|
||||
session.commit()
|
||||
|
||||
|
||||
# Should only get NVDA alert
|
||||
alerts = correlator.get_alerts_before_trade(trade, lookback_days=30)
|
||||
|
||||
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0].ticker == "NVDA"
|
||||
|
||||
|
||||
+10
-5
@@ -5,7 +5,6 @@ Tests for database models.
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from pote.db.models import Price, Security, Trade
|
||||
@@ -57,9 +56,12 @@ def test_unique_constraints(test_db_session, sample_security):
|
||||
dup_security = Security(ticker="AAPL", name="Apple Duplicate")
|
||||
test_db_session.add(dup_security)
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
try:
|
||||
test_db_session.commit()
|
||||
test_db_session.rollback()
|
||||
assert False, "Should have raised IntegrityError"
|
||||
except IntegrityError:
|
||||
test_db_session.rollback()
|
||||
# Expected behavior
|
||||
|
||||
|
||||
def test_price_unique_per_security_date(test_db_session, sample_security):
|
||||
@@ -81,9 +83,12 @@ def test_price_unique_per_security_date(test_db_session, sample_security):
|
||||
)
|
||||
test_db_session.add(price2)
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
try:
|
||||
test_db_session.commit()
|
||||
test_db_session.rollback()
|
||||
assert False, "Should have raised IntegrityError"
|
||||
except IntegrityError:
|
||||
test_db_session.rollback()
|
||||
# Expected behavior
|
||||
|
||||
|
||||
def test_trade_queries(test_db_session, sample_official, sample_security):
|
||||
|
||||
+83
-113
@@ -1,26 +1,25 @@
|
||||
"""Tests for market monitoring module."""
|
||||
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
import pytest
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from pote.db.models import MarketAlert, Official, Security, Trade
|
||||
from pote.monitoring.alert_manager import AlertManager
|
||||
from pote.monitoring.market_monitor import MarketMonitor
|
||||
from pote.monitoring.alert_manager import AlertManager
|
||||
from pote.db.models import Official, Security, Trade, MarketAlert
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_congressional_trades(test_db_session):
|
||||
"""Create sample congressional trades for watchlist building."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
# Create officials
|
||||
pelosi = Official(name="Nancy Pelosi", chamber="House", party="Democrat", state="CA")
|
||||
tuberville = Official(name="Tommy Tuberville", chamber="Senate", party="Republican", state="AL")
|
||||
session.add_all([pelosi, tuberville])
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create securities
|
||||
nvda = Security(ticker="NVDA", name="NVIDIA Corporation", sector="Technology")
|
||||
msft = Security(ticker="MSFT", name="Microsoft Corporation", sector="Technology")
|
||||
@@ -29,58 +28,28 @@ def sample_congressional_trades(test_db_session):
|
||||
spy = Security(ticker="SPY", name="SPDR S&P 500 ETF", sector="Financial")
|
||||
session.add_all([nvda, msft, aapl, tsla, spy])
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create multiple trades (NVDA is most traded)
|
||||
trades = [
|
||||
Trade(
|
||||
official_id=pelosi.id,
|
||||
security_id=nvda.id,
|
||||
source="test",
|
||||
transaction_date=date(2024, 1, 15),
|
||||
side="buy",
|
||||
value_min=Decimal("15001"),
|
||||
value_max=Decimal("50000"),
|
||||
),
|
||||
Trade(
|
||||
official_id=pelosi.id,
|
||||
security_id=nvda.id,
|
||||
source="test",
|
||||
transaction_date=date(2024, 2, 1),
|
||||
side="buy",
|
||||
value_min=Decimal("15001"),
|
||||
value_max=Decimal("50000"),
|
||||
),
|
||||
Trade(
|
||||
official_id=tuberville.id,
|
||||
security_id=nvda.id,
|
||||
source="test",
|
||||
transaction_date=date(2024, 2, 15),
|
||||
side="buy",
|
||||
value_min=Decimal("50001"),
|
||||
value_max=Decimal("100000"),
|
||||
),
|
||||
Trade(
|
||||
official_id=pelosi.id,
|
||||
security_id=msft.id,
|
||||
source="test",
|
||||
transaction_date=date(2024, 1, 20),
|
||||
side="sell",
|
||||
value_min=Decimal("15001"),
|
||||
value_max=Decimal("50000"),
|
||||
),
|
||||
Trade(
|
||||
official_id=tuberville.id,
|
||||
security_id=aapl.id,
|
||||
source="test",
|
||||
transaction_date=date(2024, 2, 10),
|
||||
side="buy",
|
||||
value_min=Decimal("15001"),
|
||||
value_max=Decimal("50000"),
|
||||
),
|
||||
Trade(official_id=pelosi.id, security_id=nvda.id, source="test",
|
||||
transaction_date=date(2024, 1, 15), side="buy",
|
||||
value_min=Decimal("15001"), value_max=Decimal("50000")),
|
||||
Trade(official_id=pelosi.id, security_id=nvda.id, source="test",
|
||||
transaction_date=date(2024, 2, 1), side="buy",
|
||||
value_min=Decimal("15001"), value_max=Decimal("50000")),
|
||||
Trade(official_id=tuberville.id, security_id=nvda.id, source="test",
|
||||
transaction_date=date(2024, 2, 15), side="buy",
|
||||
value_min=Decimal("50001"), value_max=Decimal("100000")),
|
||||
Trade(official_id=pelosi.id, security_id=msft.id, source="test",
|
||||
transaction_date=date(2024, 1, 20), side="sell",
|
||||
value_min=Decimal("15001"), value_max=Decimal("50000")),
|
||||
Trade(official_id=tuberville.id, security_id=aapl.id, source="test",
|
||||
transaction_date=date(2024, 2, 10), side="buy",
|
||||
value_min=Decimal("15001"), value_max=Decimal("50000")),
|
||||
]
|
||||
session.add_all(trades)
|
||||
session.commit()
|
||||
|
||||
|
||||
return {
|
||||
"officials": [pelosi, tuberville],
|
||||
"securities": [nvda, msft, aapl, tsla, spy],
|
||||
@@ -92,9 +61,9 @@ def sample_congressional_trades(test_db_session):
|
||||
def sample_alerts(test_db_session):
|
||||
"""Create sample market alerts."""
|
||||
session = test_db_session
|
||||
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
alerts = [
|
||||
MarketAlert(
|
||||
ticker="NVDA",
|
||||
@@ -127,10 +96,10 @@ def sample_alerts(test_db_session):
|
||||
severity=5,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
session.add_all(alerts)
|
||||
session.commit()
|
||||
|
||||
|
||||
return alerts
|
||||
|
||||
|
||||
@@ -138,9 +107,9 @@ def test_get_congressional_watchlist(test_db_session, sample_congressional_trade
|
||||
"""Test building watchlist from congressional trades."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
watchlist = monitor.get_congressional_watchlist(limit=10)
|
||||
|
||||
|
||||
assert len(watchlist) > 0
|
||||
assert "NVDA" in watchlist # Most traded
|
||||
assert watchlist[0] == "NVDA" # Should be first (3 trades)
|
||||
@@ -150,11 +119,11 @@ def test_check_ticker_basic(test_db_session):
|
||||
"""Test basic ticker checking (may not find alerts with real data)."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
# This uses real yfinance data, so alerts depend on current market
|
||||
# We test that it doesn't crash
|
||||
alerts = monitor.check_ticker("AAPL", lookback_days=5)
|
||||
|
||||
|
||||
assert isinstance(alerts, list)
|
||||
# Each alert should have required fields
|
||||
for alert in alerts:
|
||||
@@ -168,7 +137,7 @@ def test_scan_watchlist_with_mock(test_db_session, sample_congressional_trades,
|
||||
"""Test scanning watchlist with mocked data."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
# Mock the check_ticker method to return controlled data
|
||||
def mock_check_ticker(ticker, lookback_days=5):
|
||||
if ticker == "NVDA":
|
||||
@@ -176,7 +145,7 @@ def test_scan_watchlist_with_mock(test_db_session, sample_congressional_trades,
|
||||
{
|
||||
"ticker": ticker,
|
||||
"alert_type": "unusual_volume",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {"multiplier": 3.5},
|
||||
"price": Decimal("500.00"),
|
||||
"volume": 100000000,
|
||||
@@ -185,12 +154,12 @@ def test_scan_watchlist_with_mock(test_db_session, sample_congressional_trades,
|
||||
}
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
monkeypatch.setattr(monitor, "check_ticker", mock_check_ticker)
|
||||
|
||||
|
||||
# Scan with limited watchlist
|
||||
alerts = monitor.scan_watchlist(tickers=["NVDA", "MSFT"], lookback_days=5)
|
||||
|
||||
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0]["ticker"] == "NVDA"
|
||||
assert alerts[0]["alert_type"] == "unusual_volume"
|
||||
@@ -200,12 +169,12 @@ def test_save_alerts(test_db_session):
|
||||
"""Test saving alerts to database."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
alerts_data = [
|
||||
{
|
||||
"ticker": "TSLA",
|
||||
"alert_type": "price_spike",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {"change_pct": 7.5},
|
||||
"price": Decimal("250.00"),
|
||||
"volume": 75000000,
|
||||
@@ -215,7 +184,7 @@ def test_save_alerts(test_db_session):
|
||||
{
|
||||
"ticker": "TSLA",
|
||||
"alert_type": "unusual_volume",
|
||||
"timestamp": datetime.now(UTC),
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
"details": {"multiplier": 4.0},
|
||||
"price": Decimal("250.00"),
|
||||
"volume": 120000000,
|
||||
@@ -223,11 +192,11 @@ def test_save_alerts(test_db_session):
|
||||
"severity": 9,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
saved_count = monitor.save_alerts(alerts_data)
|
||||
|
||||
|
||||
assert saved_count == 2
|
||||
|
||||
|
||||
# Verify in database
|
||||
alerts = session.query(MarketAlert).filter_by(ticker="TSLA").all()
|
||||
assert len(alerts) == 2
|
||||
@@ -237,21 +206,21 @@ def test_get_recent_alerts(test_db_session, sample_alerts):
|
||||
"""Test querying recent alerts."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
# Get all alerts
|
||||
all_alerts = monitor.get_recent_alerts(days=1)
|
||||
assert len(all_alerts) >= 3
|
||||
|
||||
|
||||
# Filter by ticker
|
||||
nvda_alerts = monitor.get_recent_alerts(ticker="NVDA", days=1)
|
||||
assert len(nvda_alerts) == 2
|
||||
assert all(a.ticker == "NVDA" for a in nvda_alerts)
|
||||
|
||||
|
||||
# Filter by alert type
|
||||
volume_alerts = monitor.get_recent_alerts(alert_type="unusual_volume", days=1)
|
||||
assert len(volume_alerts) == 1
|
||||
assert volume_alerts[0].alert_type == "unusual_volume"
|
||||
|
||||
|
||||
# Filter by severity
|
||||
high_sev_alerts = monitor.get_recent_alerts(min_severity=6, days=1)
|
||||
assert all(a.severity >= 6 for a in high_sev_alerts)
|
||||
@@ -261,12 +230,12 @@ def test_get_ticker_alert_summary(test_db_session, sample_alerts):
|
||||
"""Test alert summary by ticker."""
|
||||
session = test_db_session
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
summary = monitor.get_ticker_alert_summary(days=1)
|
||||
|
||||
|
||||
assert "NVDA" in summary
|
||||
assert "MSFT" in summary
|
||||
|
||||
|
||||
nvda_summary = summary["NVDA"]
|
||||
assert nvda_summary["alert_count"] == 2
|
||||
assert nvda_summary["max_severity"] == 7
|
||||
@@ -277,11 +246,11 @@ def test_alert_manager_format_text(test_db_session, sample_alerts):
|
||||
"""Test text formatting of alerts."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
|
||||
alert = sample_alerts[0] # NVDA unusual volume
|
||||
|
||||
|
||||
text = alert_mgr.format_alert_text(alert)
|
||||
|
||||
|
||||
assert "NVDA" in text
|
||||
assert "UNUSUAL VOLUME" in text
|
||||
assert "Severity" in text
|
||||
@@ -292,11 +261,11 @@ def test_alert_manager_format_html(test_db_session, sample_alerts):
|
||||
"""Test HTML formatting of alerts."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
|
||||
alert = sample_alerts[0]
|
||||
|
||||
|
||||
html = alert_mgr.format_alert_html(alert)
|
||||
|
||||
|
||||
assert "<div" in html
|
||||
assert "NVDA" in html
|
||||
assert "unusual_volume" in html or "Unusual Volume" in html
|
||||
@@ -306,29 +275,29 @@ def test_alert_manager_filter_alerts(test_db_session, sample_alerts):
|
||||
"""Test filtering alerts."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
|
||||
# Filter by severity
|
||||
high_sev = alert_mgr.filter_alerts(sample_alerts, min_severity=6)
|
||||
assert len(high_sev) == 1
|
||||
assert high_sev[0].ticker == "NVDA"
|
||||
assert high_sev[0].severity == 7
|
||||
|
||||
|
||||
# Filter by ticker
|
||||
nvda_only = alert_mgr.filter_alerts(sample_alerts, min_severity=0, tickers=["NVDA"])
|
||||
assert len(nvda_only) == 2
|
||||
assert all(a.ticker == "NVDA" for a in nvda_only)
|
||||
|
||||
|
||||
# Filter by alert type
|
||||
volume_only = alert_mgr.filter_alerts(sample_alerts, alert_types=["unusual_volume"])
|
||||
assert len(volume_only) == 1
|
||||
assert volume_only[0].alert_type == "unusual_volume"
|
||||
|
||||
|
||||
# Combined filters
|
||||
filtered = alert_mgr.filter_alerts(
|
||||
sample_alerts,
|
||||
min_severity=4,
|
||||
tickers=["NVDA"],
|
||||
alert_types=["unusual_volume", "price_spike"],
|
||||
alert_types=["unusual_volume", "price_spike"]
|
||||
)
|
||||
assert len(filtered) == 2
|
||||
|
||||
@@ -337,9 +306,9 @@ def test_alert_manager_generate_summary_text(test_db_session, sample_alerts):
|
||||
"""Test generating text summary report."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
report = alert_mgr.generate_summary_report(sample_alerts, output_format="text")
|
||||
|
||||
|
||||
report = alert_mgr.generate_summary_report(sample_alerts, format="text")
|
||||
|
||||
assert "MARKET ACTIVITY ALERTS" in report
|
||||
assert "3 Alerts" in report
|
||||
assert "NVDA" in report
|
||||
@@ -351,9 +320,9 @@ def test_alert_manager_generate_summary_html(test_db_session, sample_alerts):
|
||||
"""Test generating HTML summary report."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
report = alert_mgr.generate_summary_report(sample_alerts, output_format="html")
|
||||
|
||||
|
||||
report = alert_mgr.generate_summary_report(sample_alerts, format="html")
|
||||
|
||||
assert "<html>" in report
|
||||
assert "<head>" in report
|
||||
assert "Market Activity Alerts" in report
|
||||
@@ -364,20 +333,20 @@ def test_alert_manager_empty_alerts(test_db_session):
|
||||
"""Test handling empty alert list."""
|
||||
session = test_db_session
|
||||
alert_mgr = AlertManager(session)
|
||||
|
||||
report = alert_mgr.generate_summary_report([], output_format="text")
|
||||
|
||||
|
||||
report = alert_mgr.generate_summary_report([], format="text")
|
||||
|
||||
assert "No alerts" in report
|
||||
|
||||
|
||||
def test_market_alert_model(test_db_session):
|
||||
"""Test MarketAlert model creation and retrieval."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
alert = MarketAlert(
|
||||
ticker="GOOGL",
|
||||
alert_type="price_spike",
|
||||
timestamp=datetime.now(UTC),
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
details={"test": "data"},
|
||||
price=Decimal("140.50"),
|
||||
volume=25000000,
|
||||
@@ -385,13 +354,13 @@ def test_market_alert_model(test_db_session):
|
||||
severity=7,
|
||||
source="test",
|
||||
)
|
||||
|
||||
|
||||
session.add(alert)
|
||||
session.commit()
|
||||
|
||||
|
||||
# Retrieve
|
||||
retrieved = session.query(MarketAlert).filter_by(ticker="GOOGL").first()
|
||||
|
||||
|
||||
assert retrieved is not None
|
||||
assert retrieved.ticker == "GOOGL"
|
||||
assert retrieved.alert_type == "price_spike"
|
||||
@@ -403,9 +372,9 @@ def test_market_alert_model(test_db_session):
|
||||
def test_alert_timestamp_filtering(test_db_session):
|
||||
"""Test filtering alerts by timestamp."""
|
||||
session = test_db_session
|
||||
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Create alerts at different times
|
||||
old_alert = MarketAlert(
|
||||
ticker="TEST1",
|
||||
@@ -419,19 +388,20 @@ def test_alert_timestamp_filtering(test_db_session):
|
||||
timestamp=now - timedelta(hours=2),
|
||||
severity=5,
|
||||
)
|
||||
|
||||
|
||||
session.add_all([old_alert, recent_alert])
|
||||
session.commit()
|
||||
|
||||
|
||||
monitor = MarketMonitor(session)
|
||||
|
||||
|
||||
# Should only get recent alert
|
||||
alerts_1_day = monitor.get_recent_alerts(days=1)
|
||||
test_alerts = [a for a in alerts_1_day if a.ticker.startswith("TEST")]
|
||||
assert len(test_alerts) == 1
|
||||
assert test_alerts[0].ticker == "TEST2"
|
||||
|
||||
|
||||
# Should get both with longer lookback
|
||||
alerts_30_days = monitor.get_recent_alerts(days=30)
|
||||
test_alerts = [a for a in alerts_30_days if a.ticker.startswith("TEST")]
|
||||
assert len(test_alerts) == 2
|
||||
|
||||
|
||||
@@ -1,39 +1,38 @@
|
||||
"""Tests for pattern detection module."""
|
||||
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
import pytest
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from pote.db.models import MarketAlert, Official, Security, Trade
|
||||
from pote.monitoring.pattern_detector import PatternDetector
|
||||
from pote.db.models import Official, Security, Trade, MarketAlert
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multiple_officials_with_patterns(test_db_session):
|
||||
"""Create multiple officials with different timing patterns."""
|
||||
session = test_db_session
|
||||
|
||||
|
||||
# Create officials
|
||||
pelosi = Official(name="Nancy Pelosi", chamber="House", party="Democrat", state="CA")
|
||||
tuberville = Official(name="Tommy Tuberville", chamber="Senate", party="Republican", state="AL")
|
||||
clean_trader = Official(name="Clean Trader", chamber="House", party="Independent", state="TX")
|
||||
|
||||
|
||||
session.add_all([pelosi, tuberville, clean_trader])
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create securities
|
||||
nvda = Security(ticker="NVDA", name="NVIDIA", sector="Technology")
|
||||
msft = Security(ticker="MSFT", name="Microsoft", sector="Technology")
|
||||
xom = Security(ticker="XOM", name="Exxon", sector="Energy")
|
||||
|
||||
|
||||
session.add_all([nvda, msft, xom])
|
||||
session.flush()
|
||||
|
||||
|
||||
# Pelosi - Suspicious pattern (trades with alerts)
|
||||
for i in range(5):
|
||||
trade_date = date(2024, 1, 15) + timedelta(days=i * 30)
|
||||
|
||||
trade_date = date(2024, 1, 15) + timedelta(days=i*30)
|
||||
|
||||
# Create trade
|
||||
trade = Trade(
|
||||
official_id=pelosi.id,
|
||||
@@ -46,23 +45,24 @@ def multiple_officials_with_patterns(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Create alerts BEFORE trade (suspicious)
|
||||
for j in range(2):
|
||||
alert = MarketAlert(
|
||||
ticker="NVDA",
|
||||
alert_type="unusual_volume",
|
||||
timestamp=datetime.combine(
|
||||
trade_date - timedelta(days=3 + j), datetime.min.time()
|
||||
).replace(tzinfo=UTC),
|
||||
trade_date - timedelta(days=3+j),
|
||||
datetime.min.time()
|
||||
).replace(tzinfo=timezone.utc),
|
||||
severity=7 + j,
|
||||
)
|
||||
session.add(alert)
|
||||
|
||||
|
||||
# Tuberville - Mixed pattern
|
||||
for i in range(4):
|
||||
trade_date = date(2024, 2, 1) + timedelta(days=i * 30)
|
||||
|
||||
trade_date = date(2024, 2, 1) + timedelta(days=i*30)
|
||||
|
||||
trade = Trade(
|
||||
official_id=tuberville.id,
|
||||
security_id=msft.id,
|
||||
@@ -74,23 +74,24 @@ def multiple_officials_with_patterns(test_db_session):
|
||||
)
|
||||
session.add(trade)
|
||||
session.flush()
|
||||
|
||||
|
||||
# Only first 2 trades have alerts
|
||||
if i < 2:
|
||||
alert = MarketAlert(
|
||||
ticker="MSFT",
|
||||
alert_type="price_spike",
|
||||
timestamp=datetime.combine(
|
||||
trade_date - timedelta(days=5), datetime.min.time()
|
||||
).replace(tzinfo=UTC),
|
||||
trade_date - timedelta(days=5),
|
||||
datetime.min.time()
|
||||
).replace(tzinfo=timezone.utc),
|
||||
severity=6,
|
||||
)
|
||||
session.add(alert)
|
||||
|
||||
|
||||
# Clean trader - No suspicious activity
|
||||
for i in range(3):
|
||||
trade_date = date(2024, 3, 1) + timedelta(days=i * 30)
|
||||
|
||||
trade_date = date(2024, 3, 1) + timedelta(days=i*30)
|
||||
|
||||
trade = Trade(
|
||||
official_id=clean_trader.id,
|
||||
security_id=xom.id,
|
||||
@@ -101,9 +102,9 @@ def multiple_officials_with_patterns(test_db_session):
|
||||
value_max=Decimal("50000"),
|
||||
)
|
||||
session.add(trade)
|
||||
|
||||
|
||||
session.commit()
|
||||
|
||||
|
||||
return {
|
||||
"officials": [pelosi, tuberville, clean_trader],
|
||||
"securities": [nvda, msft, xom],
|
||||
@@ -114,15 +115,15 @@ def test_rank_officials_by_timing(test_db_session, multiple_officials_with_patte
|
||||
"""Test ranking officials by timing scores."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
rankings = detector.rank_officials_by_timing(lookback_days=3650, min_trades=3)
|
||||
|
||||
|
||||
assert len(rankings) >= 2 # At least 2 officials with 3+ trades
|
||||
|
||||
|
||||
# Rankings should be sorted by avg_timing_score (descending)
|
||||
for i in range(len(rankings) - 1):
|
||||
assert rankings[i]["avg_timing_score"] >= rankings[i + 1]["avg_timing_score"]
|
||||
|
||||
|
||||
# Check required fields
|
||||
for ranking in rankings:
|
||||
assert "name" in ranking
|
||||
@@ -137,15 +138,16 @@ def test_identify_repeat_offenders(test_db_session, multiple_officials_with_patt
|
||||
"""Test identifying repeat offenders."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
# Set low threshold to catch Pelosi (who has 100% suspicious rate)
|
||||
offenders = detector.identify_repeat_offenders(
|
||||
lookback_days=3650, min_suspicious_rate=0.7 # 70%+
|
||||
lookback_days=3650,
|
||||
min_suspicious_rate=0.7 # 70%+
|
||||
)
|
||||
|
||||
|
||||
# Should find at least Pelosi (all trades with alerts)
|
||||
assert isinstance(offenders, list)
|
||||
|
||||
|
||||
# All offenders should have high suspicious rates
|
||||
for offender in offenders:
|
||||
assert offender["suspicious_rate"] >= 70
|
||||
@@ -155,16 +157,19 @@ def test_analyze_ticker_patterns(test_db_session, multiple_officials_with_patter
|
||||
"""Test ticker pattern analysis."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
ticker_patterns = detector.analyze_ticker_patterns(lookback_days=3650, min_trades=3)
|
||||
|
||||
|
||||
ticker_patterns = detector.analyze_ticker_patterns(
|
||||
lookback_days=3650,
|
||||
min_trades=3
|
||||
)
|
||||
|
||||
assert isinstance(ticker_patterns, list)
|
||||
assert len(ticker_patterns) >= 1 # At least NVDA should qualify
|
||||
|
||||
|
||||
# Check sorting
|
||||
for i in range(len(ticker_patterns) - 1):
|
||||
assert ticker_patterns[i]["avg_timing_score"] >= ticker_patterns[i + 1]["avg_timing_score"]
|
||||
|
||||
|
||||
# Check fields
|
||||
for pattern in ticker_patterns:
|
||||
assert "ticker" in pattern
|
||||
@@ -177,12 +182,12 @@ def test_get_sector_timing_analysis(test_db_session, multiple_officials_with_pat
|
||||
"""Test sector timing analysis."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
sector_stats = detector.get_sector_timing_analysis(lookback_days=3650)
|
||||
|
||||
|
||||
assert isinstance(sector_stats, dict)
|
||||
assert len(sector_stats) >= 2 # Technology and Energy
|
||||
|
||||
|
||||
# Check Technology sector (should have alerts)
|
||||
if "Technology" in sector_stats:
|
||||
tech = sector_stats["Technology"]
|
||||
@@ -196,14 +201,14 @@ def test_get_party_comparison(test_db_session, multiple_officials_with_patterns)
|
||||
"""Test party comparison analysis."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
party_stats = detector.get_party_comparison(lookback_days=3650)
|
||||
|
||||
|
||||
assert isinstance(party_stats, dict)
|
||||
assert len(party_stats) >= 2 # Democrat, Republican, Independent
|
||||
|
||||
|
||||
# Check that we have data for each party
|
||||
for stats in party_stats.values():
|
||||
for party, stats in party_stats.items():
|
||||
assert "official_count" in stats
|
||||
assert "total_trades" in stats
|
||||
assert "avg_timing_score" in stats
|
||||
@@ -214,9 +219,9 @@ def test_generate_pattern_report(test_db_session, multiple_officials_with_patter
|
||||
"""Test comprehensive pattern report generation."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
report = detector.generate_pattern_report(lookback_days=3650)
|
||||
|
||||
|
||||
# Check report structure
|
||||
assert "period_days" in report
|
||||
assert "summary" in report
|
||||
@@ -225,12 +230,12 @@ def test_generate_pattern_report(test_db_session, multiple_officials_with_patter
|
||||
assert "suspicious_tickers" in report
|
||||
assert "sector_analysis" in report
|
||||
assert "party_comparison" in report
|
||||
|
||||
|
||||
# Check summary
|
||||
summary = report["summary"]
|
||||
assert summary["total_officials_analyzed"] >= 2
|
||||
assert "avg_timing_score" in summary
|
||||
|
||||
|
||||
# Check that lists are populated
|
||||
assert len(report["top_suspicious_officials"]) >= 2
|
||||
assert isinstance(report["suspicious_tickers"], list)
|
||||
@@ -240,15 +245,15 @@ def test_rank_officials_min_trades_filter(test_db_session, multiple_officials_wi
|
||||
"""Test that min_trades filter works correctly."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
# With min_trades=5, should only get Pelosi
|
||||
rankings_high = detector.rank_officials_by_timing(lookback_days=3650, min_trades=5)
|
||||
|
||||
|
||||
# With min_trades=3, should get at least 2 officials
|
||||
rankings_low = detector.rank_officials_by_timing(lookback_days=3650, min_trades=3)
|
||||
|
||||
|
||||
assert len(rankings_low) >= len(rankings_high)
|
||||
|
||||
|
||||
# All officials should meet min_trades requirement
|
||||
for ranking in rankings_high:
|
||||
assert ranking["trade_count"] >= 5
|
||||
@@ -258,17 +263,17 @@ def test_empty_data_handling(test_db_session):
|
||||
"""Test handling of empty dataset."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
# With no data, should return empty results
|
||||
rankings = detector.rank_officials_by_timing(lookback_days=30, min_trades=1)
|
||||
assert rankings == []
|
||||
|
||||
|
||||
offenders = detector.identify_repeat_offenders(lookback_days=30)
|
||||
assert offenders == []
|
||||
|
||||
|
||||
tickers = detector.analyze_ticker_patterns(lookback_days=30)
|
||||
assert tickers == []
|
||||
|
||||
|
||||
sectors = detector.get_sector_timing_analysis(lookback_days=30)
|
||||
assert sectors == {}
|
||||
|
||||
@@ -277,13 +282,13 @@ def test_ranking_score_accuracy(test_db_session, multiple_officials_with_pattern
|
||||
"""Test that rankings accurately reflect timing patterns."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
rankings = detector.rank_officials_by_timing(lookback_days=3650, min_trades=3)
|
||||
|
||||
|
||||
# Find Pelosi and Clean Trader
|
||||
pelosi_rank = next((r for r in rankings if "Pelosi" in r["name"]), None)
|
||||
clean_rank = next((r for r in rankings if "Clean" in r["name"]), None)
|
||||
|
||||
|
||||
if pelosi_rank and clean_rank:
|
||||
# Pelosi (with alerts) should have higher score than clean trader (no alerts)
|
||||
assert pelosi_rank["avg_timing_score"] > clean_rank["avg_timing_score"]
|
||||
@@ -294,9 +299,9 @@ def test_sector_stats_accuracy(test_db_session, multiple_officials_with_patterns
|
||||
"""Test sector statistics are calculated correctly."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
sector_stats = detector.get_sector_timing_analysis(lookback_days=3650)
|
||||
|
||||
|
||||
# Energy should have clean pattern (no alerts)
|
||||
if "Energy" in sector_stats:
|
||||
energy = sector_stats["Energy"]
|
||||
@@ -308,12 +313,14 @@ def test_party_stats_completeness(test_db_session, multiple_officials_with_patte
|
||||
"""Test party statistics completeness."""
|
||||
session = test_db_session
|
||||
detector = PatternDetector(session)
|
||||
|
||||
|
||||
party_stats = detector.get_party_comparison(lookback_days=3650)
|
||||
|
||||
|
||||
# Check Democrats (Pelosi)
|
||||
if "Democrat" in party_stats:
|
||||
dem = party_stats["Democrat"]
|
||||
assert dem["official_count"] >= 1
|
||||
assert dem["total_trades"] >= 5 # Pelosi has 5 trades
|
||||
assert dem["total_suspicious"] > 0 # Pelosi has suspicious trades
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user