Compare commits

..

No commits in common. "feature/remove-ansible-enhance-ci" and "main" have entirely different histories.

7 changed files with 332 additions and 2291 deletions

View File

@ -3,9 +3,8 @@ name: CI
on: on:
push: push:
branches: [main, qa, dev] branches: [main, master]
pull_request: pull_request:
branches: [main, qa, dev]
jobs: jobs:
lint-and-test: lint-and-test:
@ -69,26 +68,6 @@ jobs:
echo "Testing price loader..." echo "Testing price loader..."
python scripts/fetch_sample_prices.py || true python scripts/fetch_sample_prices.py || true
secret-scanning:
runs-on: ubuntu-latest
container:
image: zricethezav/gitleaks:latest
steps:
- name: Install Node.js for checkout action
run: |
apk add --no-cache nodejs npm curl git
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Scan for secrets
run: |
echo "🔍 Scanning for exposed secrets..."
gitleaks detect --source . --no-banner --redact --exit-code 0 || true
continue-on-error: true
security-scan: security-scan:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: container:
@ -105,13 +84,11 @@ jobs:
- name: Run safety check - name: Run safety check
run: | run: |
pip install -e . pip install -e .
echo "🔍 Checking for known vulnerabilities in dependencies..."
safety check --json || true safety check --json || true
continue-on-error: true continue-on-error: true
- name: Run bandit security scan - name: Run bandit security scan
run: | run: |
echo "🔍 Running static security analysis..."
bandit -r src/ -f json -o bandit-report.json || true bandit -r src/ -f json -o bandit-report.json || true
bandit -r src/ -f screen bandit -r src/ -f screen
continue-on-error: true continue-on-error: true
@ -123,101 +100,13 @@ jobs:
steps: steps:
- name: Install Node.js for checkout action - name: Install Node.js for checkout action
run: | run: |
apk add --no-cache nodejs npm curl git apk add --no-cache nodejs npm curl
- name: Check out code - name: Check out code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Scan dependencies - name: Scan dependencies
run: | run: trivy fs --scanners vuln --exit-code 0 .
echo "🔍 Scanning dependencies for vulnerabilities..."
trivy fs --scanners vuln --exit-code 0 .
sast-scan:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
- name: Install Semgrep
run: |
apt-get update && apt-get install -y python3 python3-pip
pip3 install semgrep
- name: Run Semgrep scan
run: |
echo "🔍 Running SAST analysis with Semgrep..."
semgrep --config=auto --error || true
continue-on-error: true
container-scan:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
- name: Install Trivy
run: |
set -e
apt-get update && apt-get install -y wget curl tar
# Use a fixed, known-good Trivy version
TRIVY_VERSION="0.58.2"
TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
echo "Installing Trivy version: ${TRIVY_VERSION}"
if ! wget --progress=bar:force "${TRIVY_URL}" -O /tmp/trivy.tar.gz 2>&1; then
echo "❌ Failed to download Trivy"
exit 1
fi
if [ ! -f /tmp/trivy.tar.gz ] || [ ! -s /tmp/trivy.tar.gz ]; then
echo "❌ Downloaded Trivy archive is missing or empty"
exit 1
fi
echo "Extracting Trivy..."
if ! tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy; then
echo "❌ Failed to extract Trivy"
exit 1
fi
mv /tmp/trivy /usr/local/bin/trivy
chmod +x /usr/local/bin/trivy
trivy --version
- name: Scan Dockerfile
run: |
if [ -f "Dockerfile" ]; then
echo "🔍 Scanning Dockerfile for vulnerabilities..."
trivy config Dockerfile || true
else
echo "No Dockerfile found, skipping scan"
fi
continue-on-error: true
- name: Scan filesystem
run: |
echo "🔍 Scanning filesystem for vulnerabilities..."
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table . || true
continue-on-error: true
docker-build-test: docker-build-test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -239,11 +128,11 @@ jobs:
- name: Test Docker image - name: Test Docker image
run: | run: |
docker run --rm pote:test python -c "import pote; print('POTE import successful')" docker run --rm pote:test python -c "import pote; print('POTE import successful')"
workflow-summary: workflow-summary:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [lint-and-test, secret-scanning, security-scan, dependency-scan, sast-scan, container-scan, docker-build-test] needs: [lint-and-test, security-scan, dependency-scan, docker-build-test]
if: always() if: always()
steps: steps:
- name: Generate workflow summary - name: Generate workflow summary
@ -255,21 +144,11 @@ jobs:
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true
echo "| 🧪 Lint & Test | ${{ needs.lint-and-test.result }} |" >> $GITHUB_STEP_SUMMARY || true echo "| 🧪 Lint & Test | ${{ needs.lint-and-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔐 Secret Scanning | ${{ needs.secret-scanning.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔍 SAST Scan | ${{ needs.sast-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true echo "| 🐳 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🐳 Container Scan | ${{ needs.container-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🐋 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true echo "" >> $GITHUB_STEP_SUMMARY || true
echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true echo "" >> $GITHUB_STEP_SUMMARY || true
echo "All security and validation checks have completed." >> $GITHUB_STEP_SUMMARY || true echo "All checks have completed. Review individual job logs for details." >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
echo "**Security Layers:**" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Secret scanning (Gitleaks)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Dependency vulnerabilities (Safety + Trivy)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Static security analysis (Bandit)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ SAST scanning (Semgrep)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Container scanning (Trivy)" >> $GITHUB_STEP_SUMMARY || true
continue-on-error: true

View File

@ -1,439 +0,0 @@
# CI Pipeline - Complete Security Suite
**Enhanced CI/CD pipeline with comprehensive security scanning layers.**
---
## 🎯 What Changed
### ❌ Removed
- **`ansible/` directory** - Moved to infrastructure repository (where it belongs)
- **`ANSIBLE_INTEGRATION.md`** - Redundant with handoff docs
### ✅ Kept (Reference Documentation)
- **`ANSIBLE_HANDOFF.md`** - Integration guide for your Ansible team
- **`ANSIBLE_TECHNICAL_REFERENCE.md`** - Exact commands, paths, procedures
- **`CUSTOMIZATION_CHECKLIST.md`** - Configuration reference
- **`MOVE_ANSIBLE_TO_SEPARATE_REPO.md`** - Migration guide
### 🚀 Enhanced
- **`.github/workflows/ci.yml`** - Added 5 new security scanning jobs
---
## 🔐 Security Layers
### 1. Secret Scanning (Gitleaks)
**Tool:** [Gitleaks](https://github.com/gitleaks/gitleaks)
**Purpose:** Detect exposed secrets in code and git history
**Scans for:**
- API keys
- Passwords
- Tokens
- Private keys
- Database credentials
**Features:**
- Scans entire git history (`fetch-depth: 0`)
- Redacted output (doesn't expose secrets in logs)
- Continues on error (won't block CI)
```yaml
container: zricethezav/gitleaks:latest
command: gitleaks detect --source . --no-banner --redact --exit-code 0
```
---
### 2. Security Scan (Safety + Bandit)
**Tools:** [Safety](https://pyup.io/safety/) + [Bandit](https://bandit.readthedocs.io/)
#### Safety - Dependency Vulnerabilities
**Purpose:** Check Python dependencies against CVE database
**Scans for:**
- Known vulnerabilities in packages
- Outdated packages with security issues
- CVE references
```bash
pip install safety
safety check --json
```
#### Bandit - Static Security Analysis
**Purpose:** Find common security issues in Python code
**Scans for:**
- SQL injection vulnerabilities
- Hardcoded passwords
- Use of `eval()`, `exec()`
- Insecure temp file usage
- Weak cryptography
```bash
pip install bandit
bandit -r src/ -f screen
```
---
### 3. Dependency Scan (Trivy)
**Tool:** [Trivy](https://github.com/aquasecurity/trivy)
**Purpose:** Comprehensive vulnerability scanner
**Scans:**
- Python packages (from `pyproject.toml`)
- System libraries
- OS packages
- CVE database
```yaml
container: aquasec/trivy:latest
command: trivy fs --scanners vuln --exit-code 0 .
```
---
### 4. SAST Scan (Semgrep)
**Tool:** [Semgrep](https://semgrep.dev/)
**Purpose:** Static Application Security Testing
**Scans for:**
- Security anti-patterns
- Code quality issues
- Language-specific vulnerabilities
- OWASP Top 10 issues
**Features:**
- Language-aware (understands Python syntax)
- Pattern-based matching
- Auto-config uses community rules
```bash
pip install semgrep
semgrep --config=auto --error
```
---
### 5. Container Scan (Trivy)
**Tool:** Trivy (filesystem mode)
**Purpose:** Scan Docker configurations and filesystem
**Scans:**
- `Dockerfile` misconfigurations
- Filesystem vulnerabilities
- HIGH/CRITICAL severity issues
**Features:**
- Config scanning (Dockerfile best practices)
- Filesystem scanning (all files)
- Severity filtering
```bash
trivy config Dockerfile
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table .
```
---
## 📊 CI Pipeline Jobs
### Complete Job List
| Job | Purpose | Tool(s) | Blocking? |
|-----|---------|---------|-----------|
| **lint-and-test** | Code quality & tests | ruff, black, mypy, pytest | ✅ Yes |
| **secret-scanning** | Exposed secrets | Gitleaks | ⚠️ No |
| **security-scan** | Python security | Safety, Bandit | ⚠️ No |
| **dependency-scan** | Dependency vulns | Trivy | ⚠️ No |
| **sast-scan** | Static analysis | Semgrep | ⚠️ No |
| **container-scan** | Container security | Trivy | ⚠️ No |
| **docker-build-test** | Docker build | Docker | ✅ Yes |
| **workflow-summary** | Status report | Native | Info |
### Blocking vs Non-Blocking
**Blocking (will fail CI):**
- `lint-and-test` - Code must pass tests
- `docker-build-test` - Docker image must build
**Non-Blocking (informational):**
- All security scans use `continue-on-error: true`
- Provides visibility without blocking development
- Can be made blocking by removing `continue-on-error`
---
## 🎨 Workflow Summary
After all jobs complete, a summary is generated:
```
## 🔍 CI Workflow Summary
### Job Results
| Job | Status |
|-----|--------|
| 🧪 Lint & Test | success |
| 🔐 Secret Scanning | success |
| 🔒 Security Scan | success |
| 📦 Dependency Scan | success |
| 🔍 SAST Scan | success |
| 🐳 Container Scan | success |
| 🐋 Docker Build | success |
### 📊 Summary
All security and validation checks have completed.
**Security Layers:**
- ✅ Secret scanning (Gitleaks)
- ✅ Dependency vulnerabilities (Safety + Trivy)
- ✅ Static security analysis (Bandit)
- ✅ SAST scanning (Semgrep)
- ✅ Container scanning (Trivy)
```
---
## 🔧 Configuration
### Gitea Secrets (Optional)
None of the security scans require secrets, but you can configure:
```bash
# Optional: For SonarQube integration (if you add it later)
SONAR_HOST_URL=https://sonar.example.com
SONAR_TOKEN=your_token_here
```
### Making Scans Blocking
To make security scans fail the build, remove `continue-on-error: true`:
```yaml
# Before (non-blocking):
- name: Scan for secrets
run: gitleaks detect --source . --no-banner --redact --exit-code 0
continue-on-error: true
# After (blocking):
- name: Scan for secrets
run: gitleaks detect --source . --no-banner --redact --exit-code 1
# Removed continue-on-error
```
---
## 📈 Comparison with Your Ansible Pipeline
### Features from Your Pipeline
| Feature | Your Ansible Pipeline | POTE Pipeline | Status |
|---------|----------------------|---------------|--------|
| Markdown linting | ✅ npm run test:markdown | ❌ N/A | Not needed (Python project) |
| Ansible validation | ✅ ansible-lint | ❌ Removed | Moved to infrastructure repo |
| Secret scanning | ✅ Gitleaks | ✅ Gitleaks | ✅ Implemented |
| Dependency scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
| SAST scan | ✅ Semgrep | ✅ Semgrep | ✅ Implemented |
| License check | ✅ license-checker (npm) | ❌ N/A | Not needed (MIT license) |
| Vault check | ✅ Ansible vault | ❌ Removed | No vault files in app repo |
| Playbook test | ✅ ansible-playbook | ❌ Removed | No playbooks in app repo |
| Container scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
| SonarQube | ✅ sonar-scanner | ❌ Not added | Can add if needed |
### What's Different
**Removed (Ansible-specific):**
- Ansible linting
- Vault validation
- Playbook syntax checks
- Markdown linting (not applicable)
**Added (Python-specific):**
- Python linting (ruff, black, mypy)
- pytest with coverage
- Safety (Python dependency CVE check)
- Bandit (Python security linter)
**Kept (Universal):**
- Secret scanning (Gitleaks)
- Dependency scanning (Trivy)
- SAST scanning (Semgrep)
- Container scanning (Trivy)
---
## 🚀 Usage
### Triggers
The pipeline runs on:
```yaml
on:
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
```
### Manual Trigger
To trigger manually (if you add `workflow_dispatch`):
```yaml
on:
workflow_dispatch: # Add this
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
```
Then trigger via Gitea UI: Actions → CI → Run workflow
---
## 📊 Viewing Results
### In Gitea
1. **Navigate to:** Repository → Actions → CI workflow
2. **Click on:** Latest run
3. **View:** Individual job logs
4. **Summary:** Scroll to bottom for workflow summary
### Locally
Run the same checks locally before pushing:
```bash
# Linting
ruff check src/ tests/
black --check src/ tests/
mypy src/
# Tests
pytest tests/ -v --cov=src/pote
# Security scans (if tools installed)
gitleaks detect --source . --no-banner
safety check
bandit -r src/
semgrep --config=auto src/
trivy fs .
```
---
## 🔄 Future Enhancements
### Optional Additions
1. **SonarQube Integration**
- Code quality metrics
- Technical debt tracking
- Requires SonarQube server
2. **License Checking**
- Scan Python dependencies for licenses
- Tool: `pip-licenses`
3. **Performance Testing**
- Benchmark critical functions
- Tool: `pytest-benchmark`
4. **Code Coverage Gates**
- Fail if coverage drops below threshold
- Already have coverage reporting
5. **Dependency Updates**
- Auto-create PRs for dependency updates
- Tool: Dependabot or Renovate
---
## 🆘 Troubleshooting
### Job Failing: secret-scanning
**Issue:** Gitleaks found exposed secrets
**Solution:**
1. Review the scan output (redacted)
2. Remove secrets from code
3. Use `.env` files (already in `.gitignore`)
4. Rotate exposed credentials
### Job Failing: security-scan
**Issue:** Safety found vulnerable dependencies
**Solution:**
1. Review `safety check` output
2. Update vulnerable packages: `pip install --upgrade <package>`
3. Update `pyproject.toml` with new versions
### Job Failing: sast-scan
**Issue:** Semgrep found security issues
**Solution:**
1. Review Semgrep output
2. Fix identified issues
3. Add `# nosemgrep` comment if false positive
### Job Failing: container-scan
**Issue:** Trivy found HIGH/CRITICAL vulnerabilities
**Solution:**
1. Review Trivy output
2. Update base image in `Dockerfile`
3. Update system packages
---
## 📝 Best Practices
### 1. Review Security Scan Results
- Don't ignore security warnings
- Investigate all HIGH/CRITICAL findings
- Keep dependencies up to date
### 2. Use Secrets Management
- Never commit secrets
- Use Gitea secrets for CI/CD
- Use `.env` files locally (in `.gitignore`)
### 3. Keep Tools Updated
- Security tools are frequently updated
- Pin versions for stability
- Update quarterly
### 4. Make Critical Scans Blocking
- Consider making secret scanning blocking
- Consider making HIGH/CRITICAL vulnerability scans blocking
---
## 📞 Support
- **CI Pipeline Issues:** Check `.github/workflows/ci.yml`
- **Security Tool Docs:**
- [Gitleaks](https://github.com/gitleaks/gitleaks)
- [Safety](https://pyup.io/safety/)
- [Bandit](https://bandit.readthedocs.io/)
- [Trivy](https://github.com/aquasecurity/trivy)
- [Semgrep](https://semgrep.dev/)
---
**Last Updated:** December 2025
**Pipeline Version:** 2.0 (Enhanced Security Suite)
**Total Security Layers:** 5

View File

@ -1,311 +0,0 @@
# POTE Customization Checklist
**Everything you need to change from generic defaults to your specific deployment.**
---
## 🔴 CRITICAL - Must Change (Security & Functionality)
### 1. Email Configuration (`.env` file)
```bash
# Current generic values:
SMTP_HOST=mail.levkin.ca # ✅ Already yours
SMTP_PORT=587 # ✅ OK (standard)
SMTP_USER=test@levkin.ca # ✅ Already yours
SMTP_PASSWORD=your_password_here # 🔴 CHANGE THIS
FROM_EMAIL=test@levkin.ca # ✅ Already yours
REPORT_RECIPIENTS=admin@localhost # 🔴 CHANGE THIS to your email
```
**Action:** Update `SMTP_PASSWORD` and `REPORT_RECIPIENTS` in `.env`
### 2. Database Password (`.env` file)
```bash
# Current generic value:
DATABASE_URL=postgresql://poteuser:changeme123@localhost:5432/potedb
# 🔴 CHANGE "changeme123" to a strong password
```
**Action:** Choose a strong password and update in:
- `.env` file
- PostgreSQL (if already created): `ALTER USER poteuser PASSWORD 'your_new_password';`
### 3. Git Repository (Ansible: `ansible/group_vars/all.yml`)
```yaml
# Current value:
pote_git_repo: "gitea@10.0.30.169:ilia/POTE.git"
# 🔴 This is YOUR Gitea repo, but verify:
# - IP address: 10.0.30.169 (is this correct?)
# - Username: ilia (is this correct?)
# - Repo name: POTE (is this correct?)
```
**Action:** Verify or update the Git repo URL
### 4. SSH Keys (Ansible: `ansible/vault.example.yml`)
```yaml
# 🔴 MUST CREATE vault.yml with your actual keys:
vault_git_ssh_key: |
-----BEGIN OPENSSH PRIVATE KEY-----
your_ssh_private_key_here # 🔴 ADD YOUR KEY
-----END OPENSSH PRIVATE KEY-----
vault_ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC..." # 🔴 ADD YOUR KEY
```
**Action:**
1. Copy `ansible/vault.example.yml``ansible/group_vars/all/vault.yml`
2. Add your SSH keys
3. Encrypt: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
### 5. Server IP Addresses (Ansible: `ansible/inventory.example.yml`)
```yaml
# Current values:
development:
hosts:
pote-dev:
ansible_host: 10.0.10.100 # 🔴 CHANGE to your dev server IP
staging:
hosts:
pote-qa:
ansible_host: 10.0.10.101 # 🔴 CHANGE to your QA server IP
production:
hosts:
pote-prod:
ansible_host: 10.0.10.95 # 🔴 CHANGE to your prod server IP (or keep if correct)
```
**Action:** Update all IP addresses to match your Proxmox LXC containers
---
## 🟡 IMPORTANT - Should Change (Gitea Secrets)
### 6. Gitea Secrets (for CI/CD pipelines)
**Location:** Gitea Web UI → Repository Settings → Secrets
```bash
# Required secrets:
DB_PASSWORD=changeme123 # 🟡 CHANGE to match your DB password
SMTP_PASSWORD=your_password_here # 🟡 CHANGE to your email password
SMTP_HOST=mail.levkin.ca # ✅ Already yours
SMTP_USER=test@levkin.ca # ✅ Already yours
FROM_EMAIL=test@levkin.ca # ✅ Already yours
# For deployment workflow (if using):
PROXMOX_SSH_KEY=<your_private_key> # 🟡 ADD your SSH private key
PROXMOX_HOST=10.0.10.95 # 🟡 CHANGE to your server IP
PROXMOX_USER=poteapp # 🟡 CHANGE if using different user
```
**Action:** Add/update secrets in Gitea:
1. Go to `https://git.levkin.ca/ilia/POTE/settings/secrets`
2. Add each secret listed above
---
## 🟢 OPTIONAL - Customize for Your Needs
### 7. Email Recipients (Multiple locations)
**`.env` file:**
```bash
REPORT_RECIPIENTS=admin@localhost # 🟢 Change to your email(s), comma-separated
```
**`scripts/automated_daily_run.sh`:**
```bash
REPORT_RECIPIENTS="${REPORT_RECIPIENTS:-admin@localhost}" # 🟢 Change default
```
**`scripts/setup_cron.sh`:**
- Will prompt you interactively for email address
**Action:** Update to your preferred email(s) for reports
### 8. Market Monitoring Tickers (`.env` and Ansible)
**`.env` file:**
```bash
MARKET_MONITOR_TICKERS=NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX
# 🟢 Customize this list based on what Congress trades most
```
**`ansible/group_vars/all.yml`:**
```yaml
market_tickers: "NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX"
# 🟢 Should match .env
```
**Action:** Research most-traded congressional stocks and update list
### 9. Alert Severity Threshold (`.env` and Ansible)
**`.env` file:**
```bash
ALERT_MIN_SEVERITY=5 # 🟢 1-10, lower = more sensitive
```
**`ansible/group_vars/all.yml`:**
```yaml
alert_severity: 5 # 🟢 Should match .env
```
**Action:** Adjust based on how many alerts you want (5 is moderate)
### 10. Cron Schedule Times (Ansible)
**`ansible/group_vars/development.yml`, `staging.yml`, `production.yml`:**
```yaml
pote_daily_report_time: "0 6" # 🟢 6:00 AM - change to your preference
pote_weekly_report_time: "0 8" # 🟢 8:00 AM Sunday - change to your preference
pote_health_check_time: "*/30 * * * *" # 🟢 Every 30 min - change to your preference
```
**Action:** Adjust times based on your timezone and preferences
### 11. Log Level (`.env` and Ansible)
**`.env` file:**
```bash
LOG_LEVEL=INFO # 🟢 DEBUG, INFO, WARNING, ERROR
```
**`ansible/group_vars/all.yml`:**
```yaml
log_level: "INFO" # 🟢 Should match .env
```
**Action:** Use `DEBUG` for development, `INFO` for production
### 12. Application User (Ansible)
**`ansible/group_vars/all.yml`:**
```yaml
appuser_name: "poteapp" # 🟢 Change if you want a different username
```
**`scripts/proxmox_setup.sh`:**
```bash
APP_USER="poteapp" # 🟢 Should match Ansible
```
**Action:** Keep as `poteapp` unless you have a specific naming convention
### 13. Database Names (Ansible - per environment)
**`ansible/group_vars/development.yml`:**
```yaml
db_name: "potedb_dev" # 🟢 OK as-is
```
**`ansible/group_vars/staging.yml`:**
```yaml
db_name: "potedb_qa" # 🟢 OK as-is
```
**`ansible/group_vars/production.yml`:**
```yaml
db_name: "potedb" # 🟢 OK as-is
```
**Action:** Keep defaults unless you have a specific naming convention
### 14. Backup Retention (Ansible)
**`ansible/roles/pote/defaults/main.yml`:**
```yaml
pote_backup_retention_days: 90 # 🟢 Adjust based on disk space
```
**Action:** Increase for production (180+ days), decrease for dev (30 days)
### 15. API Keys (`.env` - if you get paid APIs)
**`.env` file:**
```bash
QUIVERQUANT_API_KEY= # 🟢 Optional paid service
FMP_API_KEY= # 🟢 Optional paid service
```
**Action:** Add keys if you subscribe to these services (not required)
---
## 📋 Summary: Quick Action List
### Immediate (Before First Deployment)
1. ✅ Update `.env`: `SMTP_PASSWORD`, `REPORT_RECIPIENTS`, `DATABASE_URL` password
2. ✅ Create `ansible/group_vars/all/vault.yml` with your SSH keys
3. ✅ Encrypt vault: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
4. ✅ Update `ansible/inventory.yml` with your server IPs
5. ✅ Add Gitea secrets: `DB_PASSWORD`, `SMTP_PASSWORD`
### Before Production Use
6. ✅ Verify Git repo URL in `ansible/group_vars/all.yml`
7. ✅ Customize `MARKET_MONITOR_TICKERS` based on research
8. ✅ Adjust cron times for your timezone
9. ✅ Set appropriate log levels per environment
### Optional Enhancements
10. ⭐ Add paid API keys if you subscribe
11. ⭐ Adjust alert sensitivity based on testing
12. ⭐ Customize backup retention per environment
---
## 🔍 How to Find These Files
```bash
# Configuration files:
.env # Main config (not in git)
src/pote/config.py # Python config loader
# Ansible files:
ansible/inventory.yml # Server IPs (copy from .example.yml)
ansible/group_vars/all.yml # Common variables
ansible/group_vars/all/vault.yml # Secrets (create from vault.example.yml)
ansible/group_vars/development.yml # Dev environment
ansible/group_vars/staging.yml # QA environment
ansible/group_vars/production.yml # Prod environment
ansible/roles/pote/defaults/main.yml # All default variables
# Scripts:
scripts/automated_daily_run.sh # Daily automation
scripts/automated_weekly_run.sh # Weekly automation
scripts/setup_cron.sh # Cron setup
scripts/proxmox_setup.sh # Initial server setup
# CI/CD:
.github/workflows/ci.yml # CI pipeline
.github/workflows/deploy.yml # Deployment pipeline
```
---
## 🚨 Security Reminders
1. **NEVER commit `.env` to git** - it's in `.gitignore`
2. **NEVER commit unencrypted `vault.yml`** - always use `ansible-vault encrypt`
3. **NEVER put real passwords in example files** - use placeholders
4. **ALWAYS use strong passwords** - minimum 16 characters, mixed case, numbers, symbols
5. **ALWAYS use Gitea secrets** for CI/CD - never hardcode in workflow files
---
## 📞 Need Help?
- **Ansible Vault:** `ansible-vault --help`
- **Gitea Secrets:** Repository Settings → Secrets → Actions
- **Environment Variables:** See `src/pote/config.py` for all available settings
- **Testing Config:** Run `python scripts/health_check.py` after changes
---
**Last Updated:** December 2025

324
TESTING_STATUS.md Normal file
View File

@ -0,0 +1,324 @@
# POTE Testing Status Report
**Date:** December 15, 2025
**Status:** ✅ All Systems Operational - Ready for Deployment
---
## 🎯 Test Suite Summary
### **55 Tests - All Passing ✅**
```
Platform: Python 3.13.5, pytest-9.0.2
Test Duration: ~1.8 seconds
Coverage: ~85% overall
```
### Test Breakdown by Module:
| Module | Tests | Status | Coverage |
|--------|-------|--------|----------|
| **Analytics** | 18 tests | ✅ PASS | 80% |
| **Models** | 7 tests | ✅ PASS | 90% |
| **Ingestion** | 14 tests | ✅ PASS | 85% |
| **Price Loader** | 8 tests | ✅ PASS | 90% |
| **Security Enricher** | 8 tests | ✅ PASS | 85% |
---
## 📊 What's Been Tested?
### ✅ Core Database Operations
- [x] Creating and querying Officials
- [x] Creating and querying Securities
- [x] Creating and querying Trades
- [x] Price data storage and retrieval
- [x] Unique constraints and relationships
- [x] Database migrations (Alembic)
### ✅ Data Ingestion
- [x] House Stock Watcher client (with fixtures)
- [x] Trade loading from JSON
- [x] Security enrichment from yfinance
- [x] Price data fetching and storage
- [x] Idempotent operations (no duplicates)
- [x] Error handling for missing/invalid data
### ✅ Analytics Engine
- [x] Return calculations (buy trades)
- [x] Return calculations (sell trades)
- [x] Multiple time windows (30/60/90/180 days)
- [x] Benchmark comparisons (SPY, QQQ, etc.)
- [x] Abnormal returns (alpha calculations)
- [x] Official performance summaries
- [x] Sector-level analysis
- [x] Disclosure timing analysis
- [x] Top performer rankings
- [x] System-wide statistics
### ✅ Edge Cases
- [x] Missing price data handling
- [x] Trades with no exit price yet
- [x] Sell trades (inverted returns)
- [x] Disclosure lags
- [x] Duplicate prevention
- [x] Invalid date ranges
- [x] Empty result sets
---
## 🧪 Test Types
### 1. Unit Tests (Fast, Isolated)
**Location:** `tests/test_*.py` (excluding integration)
**Purpose:** Test individual functions and classes
**Database:** In-memory SQLite (fresh for each test)
**Speed:** ~0.5 seconds
**Examples:**
- `test_parse_amount_range()` - Parse trade amounts
- `test_normalize_transaction_type()` - Trade type normalization
- `test_get_or_create_security()` - Security deduplication
### 2. Integration Tests (Realistic Scenarios)
**Location:** `tests/test_analytics_integration.py`
**Purpose:** Test complete workflows with synthetic data
**Database:** In-memory SQLite with realistic price data
**Speed:** ~0.7 seconds
**Examples:**
- `test_return_calculation_with_real_data()` - Full return calc pipeline
- `test_benchmark_comparison_with_real_data()` - Alpha calculations
- `test_official_performance_summary()` - Aggregated metrics
**Scenarios Tested:**
- Nancy Pelosi buys NVDA early (strong returns)
- Tommy Tuberville buys NVDA later (good but less alpha)
- 120 days of synthetic price data (realistic trends)
- SPY benchmark comparison
- Multiple time window analysis
---
## 🔧 How to Run Tests Locally
### Quick Test
```bash
cd /home/user/Documents/code/pote
source venv/bin/activate
pytest -v
```
### With Coverage Report
```bash
pytest --cov=src/pote --cov-report=html --cov-report=term
# View: firefox htmlcov/index.html
```
### Specific Test Modules
```bash
# Just analytics
pytest tests/test_analytics.py -v
# Just integration tests
pytest tests/test_analytics_integration.py -v
# Specific test
pytest tests/test_analytics.py::test_return_calculator_basic -v
```
### Watch Mode (Re-run on changes)
```bash
pytest-watch
# or
ptw
```
---
## 🚨 Known Limitations
### 1. External API Dependency
**Issue:** House Stock Watcher API is currently DOWN
**Impact:** Can't fetch live congressional trades automatically
**Workaround:**
- Use fixtures (`scripts/ingest_from_fixtures.py`)
- Manual CSV import (`scripts/scrape_alternative_sources.py`)
- Manual entry (`scripts/add_custom_trades.py`)
### 2. Market Data Limits
**Issue:** yfinance has rate limits and occasional failures
**Impact:** Bulk price fetching may be slow
**Workaround:**
- Fetch in batches
- Add retry logic (already implemented)
- Use caching (already implemented)
### 3. No Live Trading API
**Issue:** We only use public disclosure data (inherent lag)
**Impact:** Trades are 30-45 days delayed by law
**This is expected:** POTE is for research, not real-time trading
---
## 📈 Performance Benchmarks
### Test Execution Time
- **Full suite:** 1.8 seconds
- **Unit tests only:** 0.5 seconds
- **Integration tests:** 0.7 seconds
- **Parallel execution:** ~1.0 second (with pytest-xdist)
### Database Operations
- **Create official:** < 1ms
- **Create trade:** < 1ms
- **Fetch prices (100 days):** ~50ms (in-memory)
- **Calculate returns:** ~10ms per trade
- **Aggregate metrics:** ~50ms for 100 trades
---
## 🎯 Pre-Deployment Checklist
### Before Deploying to Proxmox:
- [x] All tests passing locally
- [x] No linter errors (`make lint`)
- [x] Database migrations work (`alembic upgrade head`)
- [x] Scripts are executable and work
- [x] Environment variables documented
- [x] Sample data available for testing
- [x] Documentation up to date
### On Proxmox Container:
```bash
# 1. Pull latest code
cd ~/pote
git pull
# 2. Update dependencies
pip install -e .
# 3. Run tests
pytest -v
# 4. Run migrations
alembic upgrade head
# 5. Verify system
python ~/status.sh
# 6. Test a script
python scripts/enrich_securities.py
```
---
## 🔄 Continuous Testing
### Git Pre-Commit Hook (Optional)
```bash
#!/bin/bash
# .git/hooks/pre-commit
pytest --tb=short
if [ $? -ne 0 ]; then
echo "Tests failed. Commit aborted."
exit 1
fi
```
### CI/CD Integration (Future)
When you set up GitHub Actions or GitLab CI:
```yaml
# .github/workflows/test.yml
name: Tests
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- run: pip install -e .
- run: pytest -v --cov
```
---
## 📝 Test Maintenance
### Adding New Tests
**When to add tests:**
- Adding new features
- Fixing bugs (write test that fails, then fix)
- Before refactoring (ensure tests pass before & after)
**Where to add tests:**
- Unit tests: `tests/test_<module>.py`
- Integration tests: `tests/test_<feature>_integration.py`
**Example:**
```python
def test_new_feature(test_db_session):
"""Test description."""
session = test_db_session
# Arrange
# Act
# Assert
```
### Updating Fixtures
Fixtures are in `tests/conftest.py`:
- `test_db_session` - Fresh database
- `sample_official` - Test official
- `sample_security` - Test security (AAPL)
- `sample_trade` - Test trade
- `sample_price` - Test price record
---
## 🎉 Summary
### Current Status: **PRODUCTION READY**
**What Works:**
- ✅ All 55 tests passing
- ✅ Full analytics pipeline functional
- ✅ Database operations solid
- ✅ Data ingestion from multiple sources
- ✅ Price fetching from yfinance
- ✅ Security enrichment
- ✅ Return calculations
- ✅ Benchmark comparisons
- ✅ Performance metrics
- ✅ CLI scripts operational
**What's Missing:**
- ❌ Live congressional trade API (external issue - House Stock Watcher down)
- **Workaround:** Manual import, CSV, or alternative APIs available
**Next Steps:**
1. ✅ Tests are complete
2. ✅ Code is ready
3. ➡️ **Deploy to Proxmox** (or continue with Phase 2 features)
4. ➡️ Add more data sources
5. ➡️ Build dashboard (Phase 3)
---
## 📞 Need Help?
See:
- `LOCAL_TEST_GUIDE.md` - Detailed local testing instructions
- `QUICKSTART.md` - Usage guide for deployed system
- `docs/09_data_updates.md` - How to add/update data
- `README.md` - Project overview
**Questions about testing?**
All tests are documented with docstrings - read the test files!

View File

@ -1,681 +0,0 @@
# Branch Strategy & Multi-Environment Deployment
## Overview
This guide covers setting up a proper Git branching strategy with protected branches for Dev, QA, and Production environments, integrated with your Ansible auto-deployment system.
---
## 🌳 Branch Strategy
### Recommended Branch Structure
```
main (production)
├── qa (quality assurance/staging)
└── dev (development)
```
**Alternative naming:**
```
prod (production)
├── staging (QA)
└── develop (dev)
```
---
## 🔒 Branch Protection Rules
### In Gitea: Repository Settings → Branches
#### 1. `main` (Production) - MOST PROTECTED
**Protection Rules:**
- ✅ **Require pull request reviews** (at least 1 approval)
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Restrict who can push** (only maintainers)
- ✅ **Require signed commits** (optional but recommended)
- ✅ **Block force pushes**
- ✅ **Block deletions**
- ✅ **Require linear history** (no merge commits)
**Merge Strategy:**
- Only merge from `qa` branch
- Require successful QA testing
- Tag releases: `v1.0.0`, `v1.1.0`, etc.
#### 2. `qa` (Staging) - MODERATELY PROTECTED
**Protection Rules:**
- ✅ **Require pull request reviews** (at least 1 approval)
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Block force pushes**
- ✅ **Block deletions**
**Merge Strategy:**
- Merge from `dev` branch
- Run full test suite
- Manual QA testing required
#### 3. `dev` (Development) - LIGHTLY PROTECTED
**Protection Rules:**
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Block force pushes** (optional)
- ⚠️ Allow direct commits (for rapid development)
**Merge Strategy:**
- Feature branches merge here
- Continuous integration testing
- Auto-deploy to dev environment
---
## 📋 Gitea Branch Protection Setup
### Step-by-Step Configuration
#### 1. Create Branches
```bash
cd /home/user/Documents/code/pote
# Create dev branch from main
git checkout -b dev
git push origin dev
# Create qa branch from main
git checkout -b qa
git push origin qa
# Back to main
git checkout main
```
#### 2. Configure in Gitea UI
```
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
2. Click "Add New Rule"
For MAIN branch:
- Branch name pattern: main
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
- Whitelist: (leave empty or add specific users)
For QA branch:
- Branch name pattern: qa
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
For DEV branch:
- Branch name pattern: dev
- ✅ Require status checks
- ⚠️ Allow direct push (for development)
```
---
## 🚀 Deployment Workflow Integration
### Update Workflows for Multi-Environment
#### 1. Update CI Workflow for All Branches
**File:** `.github/workflows/ci.yml`
```yaml
name: CI
on:
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
jobs:
lint-and-test:
runs-on: ubuntu-latest
# ... existing CI jobs ...
```
#### 2. Create Environment-Specific Deployment Workflows
**File:** `.github/workflows/deploy-dev.yml`
```yaml
name: Deploy to Dev
on:
push:
branches: [dev]
workflow_dispatch:
jobs:
deploy-dev:
runs-on: ubuntu-latest
environment: development
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Deploy to Dev Server
env:
DEV_HOST: ${{ secrets.DEV_HOST }}
DEV_USER: ${{ secrets.DEV_USER }}
DEV_SSH_KEY: ${{ secrets.DEV_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_DEV }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_DEV }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$DEV_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $DEV_HOST >> ~/.ssh/known_hosts
# Deploy
ssh ${DEV_USER}@${DEV_HOST} << 'ENDSSH'
cd ~/pote-dev
git fetch origin
git checkout dev
git pull origin dev
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
```
**File:** `.github/workflows/deploy-qa.yml`
```yaml
name: Deploy to QA
on:
push:
branches: [qa]
workflow_dispatch:
jobs:
deploy-qa:
runs-on: ubuntu-latest
environment: staging
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Deploy to QA Server
env:
QA_HOST: ${{ secrets.QA_HOST }}
QA_USER: ${{ secrets.QA_USER }}
QA_SSH_KEY: ${{ secrets.QA_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_QA }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_QA }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$QA_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $QA_HOST >> ~/.ssh/known_hosts
# Deploy
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
cd ~/pote-qa
git fetch origin
git checkout qa
git pull origin qa
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
- name: Run Smoke Tests
run: |
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
cd ~/pote-qa
source venv/bin/activate
python scripts/health_check.py
ENDSSH
```
**File:** `.github/workflows/deploy-prod.yml`
```yaml
name: Deploy to Production
on:
push:
branches: [main]
workflow_dispatch:
inputs:
confirm:
description: 'Type "DEPLOY" to confirm production deployment'
required: true
jobs:
deploy-prod:
runs-on: ubuntu-latest
environment: production
if: github.event.inputs.confirm == 'DEPLOY' || github.event_name == 'push'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Create Release Tag
run: |
git tag -a "v$(date +%Y%m%d-%H%M%S)" -m "Production release"
git push origin --tags
- name: Deploy to Production
env:
PROD_HOST: ${{ secrets.PROXMOX_HOST }}
PROD_USER: ${{ secrets.PROXMOX_USER }}
PROD_SSH_KEY: ${{ secrets.PROXMOX_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$PROD_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $PROD_HOST >> ~/.ssh/known_hosts
# Backup current production
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
git tag "backup-$(date +%Y%m%d-%H%M%S)"
ENDSSH
# Deploy
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
git fetch origin
git checkout main
git pull origin main
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
- name: Health Check
run: |
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
source venv/bin/activate
python scripts/health_check.py
ENDSSH
- name: Rollback on Failure
if: failure()
run: |
echo "❌ Deployment failed, rolling back..."
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
latest_backup=$(git tag -l "backup-*" | sort -r | head -1)
git checkout "$latest_backup"
source venv/bin/activate
alembic upgrade head
ENDSSH
```
---
## 🔐 Gitea Secrets for Multi-Environment
### Organize Secrets by Environment
#### Development Secrets
```
DEV_HOST=10.0.10.100
DEV_USER=poteapp
DEV_SSH_KEY=(dev SSH key)
SMTP_PASSWORD_DEV=(dev mail password)
DB_PASSWORD_DEV=dev_password_123
```
#### QA/Staging Secrets
```
QA_HOST=10.0.10.101
QA_USER=poteapp
QA_SSH_KEY=(qa SSH key)
SMTP_PASSWORD_QA=(qa mail password)
DB_PASSWORD_QA=qa_password_123
```
#### Production Secrets
```
PROXMOX_HOST=10.0.10.95
PROXMOX_USER=poteapp
PROXMOX_SSH_KEY=(prod SSH key)
SMTP_PASSWORD=(prod mail password)
DB_PASSWORD=changeme123
```
---
## 📊 Deployment Flow Diagram
```
Developer
├─> Commit to feature branch
├─> Create PR to dev
│ │
│ ├─> CI runs (tests)
│ │
│ ├─> Merge to dev
│ │
│ └─> Auto-deploy to DEV environment
├─> Test in DEV
├─> Create PR: dev → qa
│ │
│ ├─> CI runs (tests)
│ │
│ ├─> Code review required
│ │
│ ├─> Merge to qa
│ │
│ └─> Auto-deploy to QA environment
├─> QA Testing
└─> Create PR: qa → main
├─> CI runs (tests)
├─> Code review required (2 approvals)
├─> Manual approval for prod deploy
├─> Merge to main
├─> Create release tag
└─> Deploy to PRODUCTION
```
---
## 🔄 Integration with Your Ansible System
### Option 1: Gitea Webhooks → Ansible
**Setup:**
1. **In Gitea:** Settings → Webhooks → Add Webhook
- URL: `https://your-ansible-controller/webhook/pote`
- Trigger: Push events
- Branches: `dev`, `qa`, `main`
2. **Ansible Playbook:** `deploy-pote.yml`
```yaml
---
- name: Deploy POTE based on branch
hosts: "{{ target_env }}"
vars:
branch: "{{ git_branch }}"
env: "{{ target_env }}"
tasks:
- name: Pull latest code
git:
repo: gitea@10.0.30.169:ilia/POTE.git
dest: /home/poteapp/pote
version: "{{ branch }}"
force: yes
- name: Install dependencies
pip:
requirements: /home/poteapp/pote/requirements.txt
virtualenv: /home/poteapp/pote/venv
- name: Run migrations
command: alembic upgrade head
args:
chdir: /home/poteapp/pote
environment:
DATABASE_URL: "{{ database_url }}"
- name: Update secrets
template:
src: env.j2
dest: /home/poteapp/pote/.env
mode: 0600
- name: Health check
command: python scripts/health_check.py
args:
chdir: /home/poteapp/pote
```
3. **Ansible Inventory:** `inventory.yml`
```yaml
all:
children:
development:
hosts:
dev-pote:
ansible_host: 10.0.10.100
target_env: development
git_branch: dev
database_url: postgresql://poteuser:dev_pass@localhost/potedb_dev
staging:
hosts:
qa-pote:
ansible_host: 10.0.10.101
target_env: staging
git_branch: qa
database_url: postgresql://poteuser:qa_pass@localhost/potedb_qa
production:
hosts:
prod-pote:
ansible_host: 10.0.10.95
target_env: production
git_branch: main
database_url: postgresql://poteuser:prod_pass@localhost/potedb
```
### Option 2: Gitea Actions → Ansible
**File:** `.github/workflows/ansible-deploy.yml`
```yaml
name: Ansible Deploy
on:
push:
branches: [main, qa, dev]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Determine environment
id: env
run: |
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
echo "environment=production" >> $GITHUB_OUTPUT
echo "host=10.0.10.95" >> $GITHUB_OUTPUT
elif [ "${{ github.ref }}" == "refs/heads/qa" ]; then
echo "environment=staging" >> $GITHUB_OUTPUT
echo "host=10.0.10.101" >> $GITHUB_OUTPUT
else
echo "environment=development" >> $GITHUB_OUTPUT
echo "host=10.0.10.100" >> $GITHUB_OUTPUT
fi
- name: Trigger Ansible
run: |
curl -X POST https://your-ansible-controller/api/deploy \
-H "Authorization: Bearer ${{ secrets.ANSIBLE_TOKEN }}" \
-d '{
"project": "pote",
"environment": "${{ steps.env.outputs.environment }}",
"branch": "${{ github.ref_name }}",
"commit": "${{ github.sha }}"
}'
```
---
## ✅ Complete Setup Checklist
### 1. Git Configuration
- [ ] Create `dev` branch
- [ ] Create `qa` branch
- [ ] Keep `main` branch
- [ ] Push all branches to Gitea
### 2. Gitea Branch Protection
- [ ] Protect `main` (require PR + approval + CI)
- [ ] Protect `qa` (require PR + CI)
- [ ] Configure `dev` (require CI only)
### 3. Gitea Secrets
- [ ] Add DEV environment secrets
- [ ] Add QA environment secrets
- [ ] Add PROD environment secrets
### 4. Workflows
- [ ] Update `ci.yml` for all branches
- [ ] Create `deploy-dev.yml`
- [ ] Create `deploy-qa.yml`
- [ ] Create `deploy-prod.yml`
### 5. Ansible Integration
- [ ] Configure Gitea webhooks (if using webhooks)
- [ ] Update Ansible playbooks for POTE
- [ ] Test deployment to each environment
### 6. Documentation
- [ ] Document deployment process
- [ ] Create runbook for rollbacks
- [ ] Train team on workflow
---
## 🚨 What You're Missing (Important!)
### 1. **Environment Variables per Environment**
Create separate `.env` files:
- `.env.dev`
- `.env.qa`
- `.env.prod`
**Never commit these!** Use Ansible templates or Gitea secrets.
### 2. **Database Migrations Strategy**
```bash
# Test migrations in dev first
alembic upgrade head # dev
# Then qa
alembic upgrade head # qa
# Finally prod (with backup!)
pg_dump potedb > backup.sql
alembic upgrade head # prod
```
### 3. **Rollback Strategy**
```bash
# Git rollback
git checkout <previous-commit>
# Database rollback
alembic downgrade -1
# Or restore from backup
psql potedb < backup.sql
```
### 4. **Monitoring & Alerts**
- Health checks after each deployment
- Email/Slack notifications on failure
- Automated rollback on critical errors
### 5. **Feature Flags**
Consider adding feature flags for gradual rollouts:
```python
# In config.py
FEATURE_NEW_ANALYTICS = os.getenv("FEATURE_NEW_ANALYTICS", "false") == "true"
```
### 6. **Changelog & Release Notes**
Maintain `CHANGELOG.md`:
```markdown
## [1.2.0] - 2025-12-15
### Added
- Email reporting system
- Gitea secrets integration
### Fixed
- Database connection timeout
### Changed
- Improved error handling
```
---
## 📚 Quick Reference Commands
```bash
# Create branches
git checkout -b dev && git push origin dev
git checkout -b qa && git push origin qa
# Merge dev → qa
git checkout qa
git merge dev
git push origin qa
# Merge qa → main (via PR in Gitea!)
# Don't do this directly - use Pull Request
# Check which branch you're on
git branch
# See all branches
git branch -a
# Deploy manually (if Ansible fails)
ssh poteapp@10.0.10.100 "cd ~/pote && git pull origin dev"
```
---
## 🎯 Recommended Next Steps
1. **Right now:** Create dev and qa branches
2. **Today:** Set up branch protection in Gitea
3. **This week:** Create environment-specific workflows
4. **This week:** Integrate with your Ansible system
5. **Next week:** Test full deployment flow
---
**With this setup, you'll have a professional, production-ready deployment pipeline!** 🚀

View File

@ -1,353 +0,0 @@
# ✅ Branch Strategy Setup Complete!
## 🌳 Branches Created
Your POTE repository now has three branches:
```
✅ main (production) - PROTECTED
✅ qa (staging) - Ready to protect
✅ dev (development) - Ready to protect
```
**Current status:**
- `main` is already protected (you saw the error - that's good!)
- New documentation committed to `dev` branch
- Ready to configure protection for `qa` and `dev`
---
## 🔒 Next Steps: Configure Branch Protection
### Go to Gitea: https://git.levkin.ca/ilia/POTE/settings/branches
### 1. Protect `main` (Production) - Already Done! ✅
Your `main` branch is already protected (we couldn't push directly to it).
**Verify settings:**
- Branch name pattern: `main`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require approvals: 1 (or 2 for production)
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
### 2. Protect `qa` (Staging) - TODO
Click "Add New Rule":
- Branch name pattern: `qa`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ Require status checks to pass
- ✅ Block force push
- ✅ Block deletion
### 3. Configure `dev` (Development) - TODO
Click "Add New Rule":
- Branch name pattern: `dev`
- ✅ Require status checks to pass (CI must pass)
- ⚠️ Allow direct push (for rapid development)
- ✅ Block force push (optional)
---
## 📋 What You're Missing (Checklist)
### ✅ Already Have:
- [x] Three branches (main, qa, dev)
- [x] Main branch protection
- [x] Comprehensive documentation
- [x] CI/CD pipeline
- [x] Gitea secrets integration
### 🔲 Need to Add:
#### 1. **Environment-Specific Secrets in Gitea**
Go to: https://git.levkin.ca/ilia/POTE/settings/secrets
**Development:**
```
DEV_HOST=10.0.10.100 (or your dev server IP)
DEV_USER=poteapp
DEV_SSH_KEY=(SSH key for dev server)
SMTP_PASSWORD_DEV=(dev email password)
DB_PASSWORD_DEV=dev_password_123
```
**QA/Staging:**
```
QA_HOST=10.0.10.101 (or your QA server IP)
QA_USER=poteapp
QA_SSH_KEY=(SSH key for QA server)
SMTP_PASSWORD_QA=(qa email password)
DB_PASSWORD_QA=qa_password_123
```
**Production:**
```
PROXMOX_HOST=10.0.10.95 (already have this)
PROXMOX_USER=poteapp
PROXMOX_SSH_KEY=(already have this)
SMTP_PASSWORD=(already have this)
DB_PASSWORD=changeme123
```
#### 2. **Create Environment-Specific Deployment Workflows**
Files to create:
- `.github/workflows/deploy-dev.yml` (see docs/14_branch_strategy_and_deployment.md)
- `.github/workflows/deploy-qa.yml`
- `.github/workflows/deploy-prod.yml` (already have deploy.yml, can rename/update)
#### 3. **Set Up Separate Servers/Containers**
You need three environments:
| Environment | Server/Container | Database | Purpose |
|-------------|------------------|----------|---------|
| **Dev** | `10.0.10.100` (or new LXC) | `potedb_dev` | Development testing |
| **QA** | `10.0.10.101` (or new LXC) | `potedb_qa` | Pre-production testing |
| **Prod** | `10.0.10.95` (existing) | `potedb` | Production |
**Options:**
- Create 2 more LXC containers (recommended)
- Use same server with different ports/databases
- Use Docker containers
#### 4. **Ansible Integration**
**Option A: Gitea Webhooks**
```
Gitea → Settings → Webhooks → Add Webhook
URL: https://your-ansible-controller/webhook/pote
Trigger on: Push events
Branches: dev, qa, main
```
**Option B: Gitea Actions calls Ansible**
```yaml
# In workflow
- name: Trigger Ansible
run: |
curl -X POST https://ansible-controller/api/deploy \
-d '{"branch": "${{ github.ref_name }}"}'
```
#### 5. **Update Ansible Playbook**
Your Ansible playbook should:
```yaml
- name: Deploy POTE
hosts: "{{ target_env }}"
vars:
branch: "{{ git_branch }}" # dev, qa, or main
tasks:
- git:
repo: gitea@10.0.30.169:ilia/POTE.git
dest: /home/poteapp/pote
version: "{{ branch }}"
# ... rest of deployment
```
#### 6. **Database Migration Strategy**
```bash
# Always test in dev first
ssh poteapp@dev-server "cd ~/pote && alembic upgrade head"
# Then QA
ssh poteapp@qa-server "cd ~/pote && alembic upgrade head"
# Finally prod (with backup!)
ssh poteapp@prod-server "pg_dump potedb > backup.sql && cd ~/pote && alembic upgrade head"
```
#### 7. **Monitoring & Alerts**
Add to each deployment:
```yaml
- name: Health Check
run: python scripts/health_check.py
- name: Send Alert on Failure
if: failure()
run: |
# Send email/Slack notification
```
#### 8. **Environment Variables**
Create separate configs:
- `.env.dev` (in dev server)
- `.env.qa` (in qa server)
- `.env` (in prod server - already have)
**Never commit these!** Use Ansible templates or deployment workflows.
---
## 🚀 Workflow After Setup
### Development Flow:
```bash
# 1. Work on feature
git checkout dev
git pull origin dev
# ... make changes ...
git commit -m "Add feature"
git push origin dev
# 2. Auto-deploys to DEV server
# (via Gitea webhook or Actions)
# 3. Test in DEV
# 4. Promote to QA
# Create PR: dev → qa in Gitea UI
# Merge after approval
# Auto-deploys to QA server
# 5. QA Testing
# 6. Promote to PROD
# Create PR: qa → main in Gitea UI
# Requires 2 approvals
# Merge
# Manual deployment trigger (with confirmation)
```
---
## 📚 Documentation
**Main guide:** `docs/14_branch_strategy_and_deployment.md`
Covers:
- ✅ Branch protection setup
- ✅ Multi-environment workflows
- ✅ Ansible integration
- ✅ Deployment flow
- ✅ Rollback procedures
- ✅ Complete checklist
---
## 🎯 Quick Actions (Do These Now)
### 1. Configure Branch Protection (5 minutes)
```
https://git.levkin.ca/ilia/POTE/settings/branches
- Add rule for 'qa'
- Add rule for 'dev'
```
### 2. Add Environment Secrets (10 minutes)
```
https://git.levkin.ca/ilia/POTE/settings/secrets
- Add DEV_* secrets
- Add QA_* secrets
- Verify PROD secrets exist
```
### 3. Create PR for Documentation (2 minutes)
```
https://git.levkin.ca/ilia/POTE/compare/main...dev
- Create pull request
- Title: "Add branch strategy documentation"
- Merge to main
```
### 4. Decide on Server Setup
**Option 1:** Create 2 more LXC containers
```bash
# On Proxmox host
pct clone 100 101 --hostname pote-dev
pct clone 100 102 --hostname pote-qa
```
**Option 2:** Use existing server with different databases
```bash
# On existing server
createdb potedb_dev
createdb potedb_qa
```
### 5. Configure Ansible
Update your Ansible inventory to include:
- `pote-dev` host
- `pote-qa` host
- `pote-prod` host (existing)
---
## ⚠️ Important Notes
### Main Branch is Protected!
You saw this error:
```
remote: Gitea: Not allowed to push to protected branch main
```
**This is GOOD!** It means:
- ✅ Main branch is protected
- ✅ Can't accidentally push directly
- ✅ Must use Pull Requests
- ✅ Requires code review
**To update main:**
1. Push to `dev` or `qa`
2. Create Pull Request in Gitea
3. Get approval
4. Merge
### Current Branch Status
```bash
$ git branch
dev ← New documentation is here
* main ← Protected, can't push directly
qa ← Empty, same as main
```
---
## 🔗 Links
- **Repository:** https://git.levkin.ca/ilia/POTE
- **Branch Protection:** https://git.levkin.ca/ilia/POTE/settings/branches
- **Secrets:** https://git.levkin.ca/ilia/POTE/settings/secrets
- **Actions:** https://git.levkin.ca/ilia/POTE/actions
- **Create PR:** https://git.levkin.ca/ilia/POTE/compare/main...dev
---
## ✅ Summary
**What's Done:**
- ✅ Created `dev`, `qa`, `main` branches
- ✅ Main branch is protected
- ✅ Documentation committed to `dev`
- ✅ Ready for Ansible integration
**What's Next:**
1. Configure branch protection for `qa` and `dev`
2. Add environment-specific secrets
3. Create PR to merge docs to main
4. Set up dev/qa servers
5. Configure Ansible for multi-environment
6. Test deployment flow
**You're 80% there! Just need to configure Gitea settings and set up the additional servers.** 🚀

View File

@ -1,378 +0,0 @@
# 🔧 Pipeline Setup Guide for Branch Protection
## ❓ Do You Need a Pipeline?
**YES!** If you want to use "Require status checks" in branch protection, you need a CI pipeline.
**Good news:** You already have one! ✅
---
## ✅ What You Already Have
### CI Pipeline: `.github/workflows/ci.yml`
**Status:** ✅ Exists and working
**Runs on:** Push to `main`, `qa`, `dev` (just updated!)
**What it does:**
- Runs linters (ruff, black, mypy)
- Runs 93 tests
- Checks code quality
- Uses PostgreSQL for integration tests
---
## 🚀 Setup Order (IMPORTANT!)
### ⚠️ **DO THIS IN ORDER:**
### Step 1: Merge CI Updates to Main (FIRST!)
**Why:** Branch protection needs the CI pipeline to exist in the branch you're protecting.
**How:**
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
2. Click "New Pull Request"
3. Title: "Update CI for multi-branch support"
4. **Merge this PR** (you can merge without protection for now)
**What this does:**
- Updates CI to run on `main`, `qa`, and `dev`
- Makes CI available for branch protection
---
### Step 2: Verify CI is Working
After merging the PR, check:
1. Go to: https://git.levkin.ca/ilia/POTE/actions
2. You should see the CI workflow running
3. Wait for it to complete (green checkmark ✅)
**If CI fails:**
- Don't set up branch protection yet
- Fix the CI issues first
- Ensure tests pass
---
### Step 3: Configure Branch Protection (AFTER CI WORKS)
**Only after CI is passing**, go to:
https://git.levkin.ca/ilia/POTE/settings/branches
#### For `main` Branch (Already Protected)
**Verify these settings:**
- Branch pattern: `main`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1-2 approvals
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test` (this appears after CI runs once)
- ✅ Block force push
- ✅ Block deletion
#### For `qa` Branch
Click "Add New Rule":
- Branch pattern: `qa`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test`
- ✅ Block force push
- ✅ Block deletion
#### For `dev` Branch
Click "Add New Rule":
- Branch pattern: `dev`
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test`
- ⚠️ **Allow direct push** (no PR required for dev)
- ⚠️ Allow force push (optional, for rebasing)
---
## 🔍 What "Require Status Checks" Means
When you enable "Require status checks":
**Before merge:**
```
PR created: dev → qa
CI pipeline runs automatically
Tests must pass ✅
Only then can you merge
```
**If CI fails:**
```
PR created: dev → qa
CI pipeline runs
Tests fail ❌
Merge button is DISABLED
Must fix code and push again
```
---
## 📋 Status Checks Available
After your CI runs once, you'll see these options in branch protection:
**Available checks:**
- `CI / lint-and-test` - Main CI pipeline (93 tests)
- `CI / security-scan` - Security scanning
- `CI / dependency-scan` - Dependency vulnerabilities
- `CI / docker-build-test` - Docker build verification
**Recommended:**
- **For `main`:** Require ALL checks ✅
- **For `qa`:** Require `lint-and-test` + `security-scan`
- **For `dev`:** Require `lint-and-test` only ✅
---
## 🎯 Step-by-Step Setup (Complete)
### 1. Merge CI Updates (5 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
2. Create PR: "Update CI for multi-branch support"
3. Merge (you can approve your own PR for now)
4. Wait for CI to run on main branch
```
### 2. Check CI Status (2 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/actions
2. Click on the latest workflow run
3. Verify all jobs pass ✅
```
### 3. Configure Branch Protection (10 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
2. For main (update existing rule):
- ✅ Require status checks
- Select: CI / lint-and-test
3. Add rule for qa:
- Branch: qa
- ✅ Require PR
- ✅ Require status checks
- Select: CI / lint-and-test
4. Add rule for dev:
- Branch: dev
- ✅ Require status checks
- Select: CI / lint-and-test
- ⚠️ Allow direct push
```
---
## ⚠️ Common Issues
### Issue 1: "No status checks found"
**Cause:** CI hasn't run on that branch yet
**Fix:**
```bash
# Push something to trigger CI
git checkout dev
git commit --allow-empty -m "Trigger CI"
git push origin dev
# Wait for CI to run, then configure protection
```
### Issue 2: "Status check never completes"
**Cause:** CI is failing or stuck
**Fix:**
1. Go to Actions tab
2. Check the failing job
3. Fix the issue
4. Push again
### Issue 3: "Can't select status checks in dropdown"
**Cause:** CI workflow name doesn't match
**Fix:**
- Workflow must be named exactly: `CI`
- Job must be named: `lint-and-test`
- Already correct in your `.github/workflows/ci.yml`
---
## 🧪 Test Your Setup
### After configuring protection:
**Test 1: Try to push directly to main**
```bash
git checkout main
git commit --allow-empty -m "Test"
git push origin main
# Should fail: "Not allowed to push to protected branch"
```
**Test 2: Create PR with failing tests**
```bash
git checkout dev
# Break a test intentionally
git commit -m "Break test"
git push origin dev
# Create PR to qa
# Merge button should be disabled until CI passes
```
**Test 3: Create PR with passing tests**
```bash
git checkout dev
# Fix the test
git commit -m "Fix test"
git push origin dev
# Create PR to qa
# Merge button should be enabled after CI passes ✅
```
---
## 📊 What Happens After Setup
### Workflow with Protection:
```
Developer pushes to dev
CI runs automatically
✅ Tests pass
Developer creates PR: dev → qa
CI runs on PR
✅ Tests pass
Reviewer approves
✅ Merge button enabled
Merge to qa
CI runs on qa branch
✅ Tests pass
Auto-deploy to QA server (if configured)
```
**If tests fail at any point:**
```
CI runs
❌ Tests fail
Merge button DISABLED
Developer fixes code
Pushes again
CI runs again
Loop until tests pass ✅
```
---
## ✅ Checklist
Before configuring branch protection:
- [ ] CI workflow exists (`.github/workflows/ci.yml`) ✅
- [ ] CI runs on all branches (`main`, `qa`, `dev`) ✅
- [ ] CI has run at least once on each branch
- [ ] All tests are passing ✅
- [ ] You can see workflow runs in Actions tab
After configuring:
- [ ] `main` branch requires status checks
- [ ] `qa` branch requires status checks
- [ ] `dev` branch requires status checks
- [ ] Tested: Can't push directly to `main`
- [ ] Tested: PR merge blocked when CI fails
- [ ] Tested: PR merge allowed when CI passes
---
## 🎯 Quick Start (TL;DR)
```bash
# 1. Merge CI updates
# Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
# Create and merge PR
# 2. Wait for CI to run
# Check: https://git.levkin.ca/ilia/POTE/actions
# 3. Configure branch protection
# Go to: https://git.levkin.ca/ilia/POTE/settings/branches
# Add rules for main, qa, dev
# Enable "Require status checks"
# Select "CI / lint-and-test"
# Done! ✅
```
---
## 📚 Related Documentation
- **CI Workflow:** `.github/workflows/ci.yml`
- **Branch Strategy:** `docs/14_branch_strategy_and_deployment.md`
- **Setup Checklist:** `BRANCH_SETUP_COMPLETE.md`
- **Gitea Secrets:** `GITEA_SECRETS_GUIDE.md`
---
## 🚀 Summary
**Do you need a pipeline?**
- ✅ YES, to use "Require status checks"
- ✅ You already have one!
- ✅ Just need to merge it to main first
**Setup order:**
1. Merge CI updates to main (via PR)
2. Verify CI runs and passes
3. Configure branch protection
4. Test the protection
**After setup:**
- All branches protected by CI
- Can't merge failing code
- Professional development workflow
- Ready for Ansible integration
**You're almost there! Just merge the PR and configure protection.** 🎉