ilia/POTE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: ilia:dev
Branches Tags
ilia:main
ilia:dev
ilia:feature/remove-ansible-enhance-ci
ilia:qa
..
compare: ilia:main
Branches Tags
ilia:dev
ilia:feature/remove-ansible-enhance-ci
ilia:main
ilia:qa

No commits in common. "dev" and "main" have entirely different histories.
dev ... main

13 changed files with 334 additions and 3067 deletions
Show Stats Expand all files Collapse all files

3
.dockerignore
View File

@ -47,6 +47,5 @@ Thumbs.db
# Docs (optional - include if you want them in container)
docs/
# Keep README.md for Docker build
# *.md
*.md

327
.github/workflows/ci.yml vendored
View File

@ -3,9 +3,8 @@ name: CI
on:
push:
branches: [main, qa, dev]
branches: [main, master]
pull_request:
branches: [main, qa, dev]
jobs:
lint-and-test:
@ -27,12 +26,6 @@ jobs:
--health-retries 5
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
@ -75,37 +68,11 @@ jobs:
echo "Testing price loader..."
python scripts/fetch_sample_prices.py || true
secret-scanning:
runs-on: ubuntu-latest
container:
image: zricethezav/gitleaks:latest
steps:
- name: Install Node.js for checkout action
run: |
apk add --no-cache nodejs npm curl git
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Scan for secrets
run: |
echo "🔍 Scanning for exposed secrets..."
gitleaks detect --source . --no-banner --redact --exit-code 0 || true
continue-on-error: true
security-scan:
runs-on: ubuntu-latest
container:
image: python:3.11-bullseye
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
@ -117,13 +84,11 @@ jobs:
- name: Run safety check
run: |
pip install -e .
echo "🔍 Checking for known vulnerabilities in dependencies..."
safety check --json || true
continue-on-error: true
- name: Run bandit security scan
run: |
echo "🔍 Running static security analysis..."
bandit -r src/ -f json -o bandit-report.json || true
bandit -r src/ -f screen
continue-on-error: true
@ -135,276 +100,13 @@ jobs:
steps:
- name: Install Node.js for checkout action
run: |
apk add --no-cache nodejs npm curl git
apk add --no-cache nodejs npm curl
- name: Check out code
uses: actions/checkout@v4
- name: Scan dependencies
run: |
echo "🔍 Scanning dependencies for vulnerabilities..."
trivy fs --scanners vuln --exit-code 0 .
sast-scan:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
- name: Install Semgrep
run: |
apt-get update && apt-get install -y python3 python3-pip
pip3 install semgrep
- name: Run Semgrep scan
run: |
echo "🔍 Running SAST analysis with Semgrep..."
semgrep --config=auto --error || true
continue-on-error: true
container-scan:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
- name: Install Trivy
run: |
set -e
apt-get update && apt-get install -y wget curl tar
# Use a fixed, known-good Trivy version
TRIVY_VERSION="0.58.2"
TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
echo "Installing Trivy version: ${TRIVY_VERSION}"
if ! wget --progress=bar:force "${TRIVY_URL}" -O /tmp/trivy.tar.gz 2>&1; then
echo "❌ Failed to download Trivy"
exit 1
fi
if [ ! -f /tmp/trivy.tar.gz ] || [ ! -s /tmp/trivy.tar.gz ]; then
echo "❌ Downloaded Trivy archive is missing or empty"
exit 1
fi
echo "Extracting Trivy..."
if ! tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy; then
echo "❌ Failed to extract Trivy"
exit 1
fi
mv /tmp/trivy /usr/local/bin/trivy
chmod +x /usr/local/bin/trivy
trivy --version
- name: Scan Dockerfile
run: |
if [ -f "Dockerfile" ]; then
echo "🔍 Scanning Dockerfile for vulnerabilities..."
trivy config Dockerfile || true
else
echo "No Dockerfile found, skipping scan"
fi
continue-on-error: true
- name: Scan filesystem
run: |
echo "🔍 Scanning filesystem for vulnerabilities..."
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table . || true
continue-on-error: true
sonar-analysis:
if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/qa'
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
env:
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
steps:
- name: Install Node.js for checkout action
run: |
export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y curl
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
- name: Check out code
uses: actions/checkout@v4
- name: Install Java and SonarScanner
run: |
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y wget curl unzip openjdk-21-jre
# Use a known working version to avoid download issues
SONAR_SCANNER_VERSION="5.0.1.3006"
SCANNER_URL="https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip"
echo "Installing SonarScanner version: ${SONAR_SCANNER_VERSION}"
echo "Downloading from: ${SCANNER_URL}"
# Download with verbose error output
if ! wget --progress=bar:force "${SCANNER_URL}" -O /tmp/sonar-scanner.zip 2>&1; then
echo "❌ Failed to download SonarScanner"
echo "Checking if file was partially downloaded:"
ls -lh /tmp/sonar-scanner.zip 2>/dev/null || echo "No file found"
exit 1
fi
# Verify download
if [ ! -f /tmp/sonar-scanner.zip ] || [ ! -s /tmp/sonar-scanner.zip ]; then
echo "❌ Downloaded file is missing or empty"
exit 1
fi
echo "Download complete. File size: $(du -h /tmp/sonar-scanner.zip | cut -f1)"
echo "Extracting SonarScanner..."
if ! unzip -q /tmp/sonar-scanner.zip -d /tmp; then
echo "❌ Failed to extract SonarScanner"
echo "Archive info:"
file /tmp/sonar-scanner.zip || true
unzip -l /tmp/sonar-scanner.zip 2>&1 | head -20 || true
exit 1
fi
# Find the extracted directory (handle both naming conventions)
EXTRACTED_DIR=""
if [ -d "/tmp/sonar-scanner-${SONAR_SCANNER_VERSION}-linux" ]; then
EXTRACTED_DIR="/tmp/sonar-scanner-${SONAR_SCANNER_VERSION}-linux"
elif [ -d "/tmp/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux" ]; then
EXTRACTED_DIR="/tmp/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux"
else
# Try to find any sonar-scanner directory
EXTRACTED_DIR=$(find /tmp -maxdepth 1 -type d -name "*sonar-scanner*" | head -1)
fi
if [ -z "$EXTRACTED_DIR" ] || [ ! -d "$EXTRACTED_DIR" ]; then
echo "❌ SonarScanner directory not found after extraction"
echo "Contents of /tmp:"
ls -la /tmp/ | grep -E "(sonar|zip)" || ls -la /tmp/ | head -20
exit 1
fi
echo "Found extracted directory: ${EXTRACTED_DIR}"
mv "${EXTRACTED_DIR}" /opt/sonar-scanner
# Create symlink
if [ -f /opt/sonar-scanner/bin/sonar-scanner ]; then
ln -sf /opt/sonar-scanner/bin/sonar-scanner /usr/local/bin/sonar-scanner
chmod +x /opt/sonar-scanner/bin/sonar-scanner
chmod +x /usr/local/bin/sonar-scanner
else
echo "❌ sonar-scanner binary not found in /opt/sonar-scanner/bin/"
echo "Contents of /opt/sonar-scanner/bin/:"
ls -la /opt/sonar-scanner/bin/ || true
exit 1
fi
echo "Verifying installation..."
if ! sonar-scanner --version; then
echo "❌ SonarScanner verification failed"
echo "PATH: $PATH"
which sonar-scanner || echo "sonar-scanner not in PATH"
exit 1
fi
echo "✓ SonarScanner installed successfully"
- name: Verify SonarQube connection
run: |
echo "Checking SonarQube connectivity..."
if [ -z "$SONAR_HOST_URL" ] || [ -z "$SONAR_TOKEN" ]; then
echo "⚠️ Skipping SonarQube analysis: SONAR_HOST_URL or SONAR_TOKEN secrets are not set."
exit 0
fi
echo "✓ Secrets are configured"
echo "SonarQube URL: ${SONAR_HOST_URL}"
echo "Testing connectivity to SonarQube server..."
if curl -f -s -o /dev/null -w "%{http_code}" "${SONAR_HOST_URL}/api/system/status" | grep -q "200"; then
echo "✓ SonarQube server is reachable"
else
echo "⚠️ Warning: Could not verify SonarQube server connectivity (continuing anyway)"
fi
- name: Install Python 3.11 and dependencies
run: |
export DEBIAN_FRONTEND=noninteractive
export TZ=UTC
ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
apt-get update
apt-get install -y software-properties-common
add-apt-repository -y ppa:deadsnakes/ppa
apt-get update
apt-get install -y python3.11 python3.11-dev python3.11-venv python3.11-distutils
# Install pip for Python 3.11
curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
# Ensure python3 and pip3 point to 3.11
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 || true
# Use python3.11 -m pip for pip operations
python3.11 -m pip install --upgrade pip
python3 --version
python3.11 -m pip --version
- name: Generate coverage report
run: |
echo "Generating coverage report for SonarQube..."
python3.11 -m pip install -e ".[dev]"
python3.11 -m pytest tests/ --cov=src/pote --cov-report=xml --cov-report=term || true
- name: Run SonarScanner
run: |
echo "Starting SonarQube analysis..."
if [ -z "$SONAR_HOST_URL" ] || [ -z "$SONAR_TOKEN" ]; then
echo "Skipping SonarQube analysis: secrets not set."
exit 0
fi
if ! sonar-scanner \
-Dsonar.projectKey=pote \
-Dsonar.sources=src \
-Dsonar.tests=tests \
-Dsonar.python.coverage.reportPaths=coverage.xml \
-Dsonar.host.url=${SONAR_HOST_URL} \
-Dsonar.token=${SONAR_TOKEN} \
-Dsonar.scm.disabled=true \
-Dsonar.python.version=3.11 \
-X; then
echo ""
echo "❌ SonarScanner analysis failed!"
echo ""
echo "Common issues:"
echo " 1. Project 'pote' doesn't exist in SonarQube"
echo " → Create it manually in SonarQube UI"
echo " 2. Token doesn't have permission to analyze/create project"
echo " → Ensure token has 'Execute Analysis' permission"
echo " 3. Token doesn't have 'Create Projects' permission (if project doesn't exist)"
echo " → Grant this permission in SonarQube user settings"
echo ""
echo "Check SonarQube logs for more details."
# Do not fail CI on Sonar auth/project setup issues.
exit 0
fi
continue-on-error: true
run: trivy fs --scanners vuln --exit-code 0 .
docker-build-test:
runs-on: ubuntu-latest
@ -420,18 +122,17 @@ jobs:
with:
context: .
push: false
load: true
tags: pote:test
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Test Docker image
run: |
docker run --rm pote:test python -c "import pote; print('✅ POTE import successful')"
docker run --rm pote:test python -c "import pote; print('POTE import successful')"
workflow-summary:
runs-on: ubuntu-latest
needs: [lint-and-test, secret-scanning, security-scan, dependency-scan, sast-scan, container-scan, sonar-analysis, docker-build-test]
needs: [lint-and-test, security-scan, dependency-scan, docker-build-test]
if: always()
steps:
- name: Generate workflow summary
@ -443,23 +144,11 @@ jobs:
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true
echo "| 🧪 Lint & Test | ${{ needs.lint-and-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔐 Secret Scanning | ${{ needs.secret-scanning.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔍 SAST Scan | ${{ needs.sast-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🐳 Container Scan | ${{ needs.container-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🔍 SonarQube Analysis | ${{ needs.sonar-analysis.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🐋 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "| 🐳 Docker Build | ${{ needs.docker-build-test.result }} |" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
echo "All security and validation checks have completed." >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
echo "**Security Layers:**" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Secret scanning (Gitleaks)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Dependency vulnerabilities (Safety + Trivy)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Static security analysis (Bandit)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ SAST scanning (Semgrep)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Container scanning (Trivy)" >> $GITHUB_STEP_SUMMARY || true
echo "- ✅ Code quality analysis (SonarQube)" >> $GITHUB_STEP_SUMMARY || true
continue-on-error: true
echo "All checks have completed. Review individual job logs for details." >> $GITHUB_STEP_SUMMARY || true

439
CI_PIPELINE_COMPLETE.md
View File

@ -1,439 +0,0 @@
# CI Pipeline - Complete Security Suite
**Enhanced CI/CD pipeline with comprehensive security scanning layers.**
---
## 🎯 What Changed
### ❌ Removed
- **`ansible/` directory** - Moved to infrastructure repository (where it belongs)
- **`ANSIBLE_INTEGRATION.md`** - Redundant with handoff docs
### ✅ Kept (Reference Documentation)
- **`ANSIBLE_HANDOFF.md`** - Integration guide for your Ansible team
- **`ANSIBLE_TECHNICAL_REFERENCE.md`** - Exact commands, paths, procedures
- **`CUSTOMIZATION_CHECKLIST.md`** - Configuration reference
- **`MOVE_ANSIBLE_TO_SEPARATE_REPO.md`** - Migration guide
### 🚀 Enhanced
- **`.github/workflows/ci.yml`** - Added 5 new security scanning jobs
---
## 🔐 Security Layers
### 1. Secret Scanning (Gitleaks)
**Tool:** [Gitleaks](https://github.com/gitleaks/gitleaks)
**Purpose:** Detect exposed secrets in code and git history
**Scans for:**
- API keys
- Passwords
- Tokens
- Private keys
- Database credentials
**Features:**
- Scans entire git history (`fetch-depth: 0`)
- Redacted output (doesn't expose secrets in logs)
- Continues on error (won't block CI)
```yaml
container: zricethezav/gitleaks:latest
command: gitleaks detect --source . --no-banner --redact --exit-code 0
```
---
### 2. Security Scan (Safety + Bandit)
**Tools:** [Safety](https://pyup.io/safety/) + [Bandit](https://bandit.readthedocs.io/)
#### Safety - Dependency Vulnerabilities
**Purpose:** Check Python dependencies against CVE database
**Scans for:**
- Known vulnerabilities in packages
- Outdated packages with security issues
- CVE references
```bash
pip install safety
safety check --json
```
#### Bandit - Static Security Analysis
**Purpose:** Find common security issues in Python code
**Scans for:**
- SQL injection vulnerabilities
- Hardcoded passwords
- Use of `eval()`, `exec()`
- Insecure temp file usage
- Weak cryptography
```bash
pip install bandit
bandit -r src/ -f screen
```
---
### 3. Dependency Scan (Trivy)
**Tool:** [Trivy](https://github.com/aquasecurity/trivy)
**Purpose:** Comprehensive vulnerability scanner
**Scans:**
- Python packages (from `pyproject.toml`)
- System libraries
- OS packages
- CVE database
```yaml
container: aquasec/trivy:latest
command: trivy fs --scanners vuln --exit-code 0 .
```
---
### 4. SAST Scan (Semgrep)
**Tool:** [Semgrep](https://semgrep.dev/)
**Purpose:** Static Application Security Testing
**Scans for:**
- Security anti-patterns
- Code quality issues
- Language-specific vulnerabilities
- OWASP Top 10 issues
**Features:**
- Language-aware (understands Python syntax)
- Pattern-based matching
- Auto-config uses community rules
```bash
pip install semgrep
semgrep --config=auto --error
```
---
### 5. Container Scan (Trivy)
**Tool:** Trivy (filesystem mode)
**Purpose:** Scan Docker configurations and filesystem
**Scans:**
- `Dockerfile` misconfigurations
- Filesystem vulnerabilities
- HIGH/CRITICAL severity issues
**Features:**
- Config scanning (Dockerfile best practices)
- Filesystem scanning (all files)
- Severity filtering
```bash
trivy config Dockerfile
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table .
```
---
## 📊 CI Pipeline Jobs
### Complete Job List
| Job | Purpose | Tool(s) | Blocking? |
|-----|---------|---------|-----------|
| **lint-and-test** | Code quality & tests | ruff, black, mypy, pytest | ✅ Yes |
| **secret-scanning** | Exposed secrets | Gitleaks | ⚠️ No |
| **security-scan** | Python security | Safety, Bandit | ⚠️ No |
| **dependency-scan** | Dependency vulns | Trivy | ⚠️ No |
| **sast-scan** | Static analysis | Semgrep | ⚠️ No |
| **container-scan** | Container security | Trivy | ⚠️ No |
| **docker-build-test** | Docker build | Docker | ✅ Yes |
| **workflow-summary** | Status report | Native | ℹ️ Info |
### Blocking vs Non-Blocking
**Blocking (will fail CI):**
- `lint-and-test` - Code must pass tests
- `docker-build-test` - Docker image must build
**Non-Blocking (informational):**
- All security scans use `continue-on-error: true`
- Provides visibility without blocking development
- Can be made blocking by removing `continue-on-error`
---
## 🎨 Workflow Summary
After all jobs complete, a summary is generated:
```
## 🔍 CI Workflow Summary
### Job Results
| Job | Status |
|-----|--------|
| 🧪 Lint & Test | success |
| 🔐 Secret Scanning | success |
| 🔒 Security Scan | success |
| 📦 Dependency Scan | success |
| 🔍 SAST Scan | success |
| 🐳 Container Scan | success |
| 🐋 Docker Build | success |
### 📊 Summary
All security and validation checks have completed.
**Security Layers:**
- ✅ Secret scanning (Gitleaks)
- ✅ Dependency vulnerabilities (Safety + Trivy)
- ✅ Static security analysis (Bandit)
- ✅ SAST scanning (Semgrep)
- ✅ Container scanning (Trivy)
```
---
## 🔧 Configuration
### Gitea Secrets (Optional)
None of the security scans require secrets, but you can configure:
```bash
# Optional: For SonarQube integration (if you add it later)
SONAR_HOST_URL=https://sonar.example.com
SONAR_TOKEN=your_token_here
```
### Making Scans Blocking
To make security scans fail the build, remove `continue-on-error: true`:
```yaml
# Before (non-blocking):
- name: Scan for secrets
run: gitleaks detect --source . --no-banner --redact --exit-code 0
continue-on-error: true
# After (blocking):
- name: Scan for secrets
run: gitleaks detect --source . --no-banner --redact --exit-code 1
# Removed continue-on-error
```
---
## 📈 Comparison with Your Ansible Pipeline
### Features from Your Pipeline
| Feature | Your Ansible Pipeline | POTE Pipeline | Status |
|---------|----------------------|---------------|--------|
| Markdown linting | ✅ npm run test:markdown | ❌ N/A | Not needed (Python project) |
| Ansible validation | ✅ ansible-lint | ❌ Removed | Moved to infrastructure repo |
| Secret scanning | ✅ Gitleaks | ✅ Gitleaks | ✅ Implemented |
| Dependency scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
| SAST scan | ✅ Semgrep | ✅ Semgrep | ✅ Implemented |
| License check | ✅ license-checker (npm) | ❌ N/A | Not needed (MIT license) |
| Vault check | ✅ Ansible vault | ❌ Removed | No vault files in app repo |
| Playbook test | ✅ ansible-playbook | ❌ Removed | No playbooks in app repo |
| Container scan | ✅ Trivy | ✅ Trivy | ✅ Implemented |
| SonarQube | ✅ sonar-scanner | ❌ Not added | Can add if needed |
### What's Different
**Removed (Ansible-specific):**
- Ansible linting
- Vault validation
- Playbook syntax checks
- Markdown linting (not applicable)
**Added (Python-specific):**
- Python linting (ruff, black, mypy)
- pytest with coverage
- Safety (Python dependency CVE check)
- Bandit (Python security linter)
**Kept (Universal):**
- Secret scanning (Gitleaks)
- Dependency scanning (Trivy)
- SAST scanning (Semgrep)
- Container scanning (Trivy)
---
## 🚀 Usage
### Triggers
The pipeline runs on:
```yaml
on:
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
```
### Manual Trigger
To trigger manually (if you add `workflow_dispatch`):
```yaml
on:
workflow_dispatch: # Add this
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
```
Then trigger via Gitea UI: Actions → CI → Run workflow
---
## 📊 Viewing Results
### In Gitea
1. **Navigate to:** Repository → Actions → CI workflow
2. **Click on:** Latest run
3. **View:** Individual job logs
4. **Summary:** Scroll to bottom for workflow summary
### Locally
Run the same checks locally before pushing:
```bash
# Linting
ruff check src/ tests/
black --check src/ tests/
mypy src/
# Tests
pytest tests/ -v --cov=src/pote
# Security scans (if tools installed)
gitleaks detect --source . --no-banner
safety check
bandit -r src/
semgrep --config=auto src/
trivy fs .
```
---
## 🔄 Future Enhancements
### Optional Additions
1. **SonarQube Integration**
- Code quality metrics
- Technical debt tracking
- Requires SonarQube server
2. **License Checking**
- Scan Python dependencies for licenses
- Tool: `pip-licenses`
3. **Performance Testing**
- Benchmark critical functions
- Tool: `pytest-benchmark`
4. **Code Coverage Gates**
- Fail if coverage drops below threshold
- Already have coverage reporting
5. **Dependency Updates**
- Auto-create PRs for dependency updates
- Tool: Dependabot or Renovate
---
## 🆘 Troubleshooting
### Job Failing: secret-scanning
**Issue:** Gitleaks found exposed secrets
**Solution:**
1. Review the scan output (redacted)
2. Remove secrets from code
3. Use `.env` files (already in `.gitignore`)
4. Rotate exposed credentials
### Job Failing: security-scan
**Issue:** Safety found vulnerable dependencies
**Solution:**
1. Review `safety check` output
2. Update vulnerable packages: `pip install --upgrade <package>`
3. Update `pyproject.toml` with new versions
### Job Failing: sast-scan
**Issue:** Semgrep found security issues
**Solution:**
1. Review Semgrep output
2. Fix identified issues
3. Add `# nosemgrep` comment if false positive
### Job Failing: container-scan
**Issue:** Trivy found HIGH/CRITICAL vulnerabilities
**Solution:**
1. Review Trivy output
2. Update base image in `Dockerfile`
3. Update system packages
---
## 📝 Best Practices
### 1. Review Security Scan Results
- Don't ignore security warnings
- Investigate all HIGH/CRITICAL findings
- Keep dependencies up to date
### 2. Use Secrets Management
- Never commit secrets
- Use Gitea secrets for CI/CD
- Use `.env` files locally (in `.gitignore`)
### 3. Keep Tools Updated
- Security tools are frequently updated
- Pin versions for stability
- Update quarterly
### 4. Make Critical Scans Blocking
- Consider making secret scanning blocking
- Consider making HIGH/CRITICAL vulnerability scans blocking
---
## 📞 Support
- **CI Pipeline Issues:** Check `.github/workflows/ci.yml`
- **Security Tool Docs:**
- [Gitleaks](https://github.com/gitleaks/gitleaks)
- [Safety](https://pyup.io/safety/)
- [Bandit](https://bandit.readthedocs.io/)
- [Trivy](https://github.com/aquasecurity/trivy)
- [Semgrep](https://semgrep.dev/)
---
**Last Updated:** December 2025
**Pipeline Version:** 2.0 (Enhanced Security Suite)
**Total Security Layers:** 5

311
CUSTOMIZATION_CHECKLIST.md
View File

@ -1,311 +0,0 @@
# POTE Customization Checklist
**Everything you need to change from generic defaults to your specific deployment.**
---
## 🔴 CRITICAL - Must Change (Security & Functionality)
### 1. Email Configuration (`.env` file)
```bash
# Current generic values:
SMTP_HOST=mail.levkin.ca # ✅ Already yours
SMTP_PORT=587 # ✅ OK (standard)
SMTP_USER=test@levkin.ca # ✅ Already yours
SMTP_PASSWORD=your_password_here # 🔴 CHANGE THIS
FROM_EMAIL=test@levkin.ca # ✅ Already yours
REPORT_RECIPIENTS=admin@localhost # 🔴 CHANGE THIS to your email
```
**Action:** Update `SMTP_PASSWORD` and `REPORT_RECIPIENTS` in `.env`
### 2. Database Password (`.env` file)
```bash
# Current generic value:
DATABASE_URL=postgresql://poteuser:changeme123@localhost:5432/potedb
# 🔴 CHANGE "changeme123" to a strong password
```
**Action:** Choose a strong password and update in:
- `.env` file
- PostgreSQL (if already created): `ALTER USER poteuser PASSWORD 'your_new_password';`
### 3. Git Repository (Ansible: `ansible/group_vars/all.yml`)
```yaml
# Current value:
pote_git_repo: "gitea@10.0.30.169:ilia/POTE.git"
# 🔴 This is YOUR Gitea repo, but verify:
# - IP address: 10.0.30.169 (is this correct?)
# - Username: ilia (is this correct?)
# - Repo name: POTE (is this correct?)
```
**Action:** Verify or update the Git repo URL
### 4. SSH Keys (Ansible: `ansible/vault.example.yml`)
```yaml
# 🔴 MUST CREATE vault.yml with your actual keys:
vault_git_ssh_key: |
-----BEGIN OPENSSH PRIVATE KEY-----
your_ssh_private_key_here # 🔴 ADD YOUR KEY
-----END OPENSSH PRIVATE KEY-----
vault_ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC..." # 🔴 ADD YOUR KEY
```
**Action:**
1. Copy `ansible/vault.example.yml` → `ansible/group_vars/all/vault.yml`
2. Add your SSH keys
3. Encrypt: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
### 5. Server IP Addresses (Ansible: `ansible/inventory.example.yml`)
```yaml
# Current values:
development:
hosts:
pote-dev:
ansible_host: 10.0.10.100 # 🔴 CHANGE to your dev server IP
staging:
hosts:
pote-qa:
ansible_host: 10.0.10.101 # 🔴 CHANGE to your QA server IP
production:
hosts:
pote-prod:
ansible_host: 10.0.10.95 # 🔴 CHANGE to your prod server IP (or keep if correct)
```
**Action:** Update all IP addresses to match your Proxmox LXC containers
---
## 🟡 IMPORTANT - Should Change (Gitea Secrets)
### 6. Gitea Secrets (for CI/CD pipelines)
**Location:** Gitea Web UI → Repository Settings → Secrets
```bash
# Required secrets:
DB_PASSWORD=changeme123 # 🟡 CHANGE to match your DB password
SMTP_PASSWORD=your_password_here # 🟡 CHANGE to your email password
SMTP_HOST=mail.levkin.ca # ✅ Already yours
SMTP_USER=test@levkin.ca # ✅ Already yours
FROM_EMAIL=test@levkin.ca # ✅ Already yours
# For deployment workflow (if using):
PROXMOX_SSH_KEY=<your_private_key> # 🟡 ADD your SSH private key
PROXMOX_HOST=10.0.10.95 # 🟡 CHANGE to your server IP
PROXMOX_USER=poteapp # 🟡 CHANGE if using different user
```
**Action:** Add/update secrets in Gitea:
1. Go to `https://git.levkin.ca/ilia/POTE/settings/secrets`
2. Add each secret listed above
---
## 🟢 OPTIONAL - Customize for Your Needs
### 7. Email Recipients (Multiple locations)
**`.env` file:**
```bash
REPORT_RECIPIENTS=admin@localhost # 🟢 Change to your email(s), comma-separated
```
**`scripts/automated_daily_run.sh`:**
```bash
REPORT_RECIPIENTS="${REPORT_RECIPIENTS:-admin@localhost}" # 🟢 Change default
```
**`scripts/setup_cron.sh`:**
- Will prompt you interactively for email address
**Action:** Update to your preferred email(s) for reports
### 8. Market Monitoring Tickers (`.env` and Ansible)
**`.env` file:**
```bash
MARKET_MONITOR_TICKERS=NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX
# 🟢 Customize this list based on what Congress trades most
```
**`ansible/group_vars/all.yml`:**
```yaml
market_tickers: "NVDA,TSLA,AAPL,MSFT,GOOGL,META,AMZN,AMD,INTC,NFLX"
# 🟢 Should match .env
```
**Action:** Research most-traded congressional stocks and update list
### 9. Alert Severity Threshold (`.env` and Ansible)
**`.env` file:**
```bash
ALERT_MIN_SEVERITY=5 # 🟢 1-10, lower = more sensitive
```
**`ansible/group_vars/all.yml`:**
```yaml
alert_severity: 5 # 🟢 Should match .env
```
**Action:** Adjust based on how many alerts you want (5 is moderate)
### 10. Cron Schedule Times (Ansible)
**`ansible/group_vars/development.yml`, `staging.yml`, `production.yml`:**
```yaml
pote_daily_report_time: "0 6" # 🟢 6:00 AM - change to your preference
pote_weekly_report_time: "0 8" # 🟢 8:00 AM Sunday - change to your preference
pote_health_check_time: "*/30 * * * *" # 🟢 Every 30 min - change to your preference
```
**Action:** Adjust times based on your timezone and preferences
### 11. Log Level (`.env` and Ansible)
**`.env` file:**
```bash
LOG_LEVEL=INFO # 🟢 DEBUG, INFO, WARNING, ERROR
```
**`ansible/group_vars/all.yml`:**
```yaml
log_level: "INFO" # 🟢 Should match .env
```
**Action:** Use `DEBUG` for development, `INFO` for production
### 12. Application User (Ansible)
**`ansible/group_vars/all.yml`:**
```yaml
appuser_name: "poteapp" # 🟢 Change if you want a different username
```
**`scripts/proxmox_setup.sh`:**
```bash
APP_USER="poteapp" # 🟢 Should match Ansible
```
**Action:** Keep as `poteapp` unless you have a specific naming convention
### 13. Database Names (Ansible - per environment)
**`ansible/group_vars/development.yml`:**
```yaml
db_name: "potedb_dev" # 🟢 OK as-is
```
**`ansible/group_vars/staging.yml`:**
```yaml
db_name: "potedb_qa" # 🟢 OK as-is
```
**`ansible/group_vars/production.yml`:**
```yaml
db_name: "potedb" # 🟢 OK as-is
```
**Action:** Keep defaults unless you have a specific naming convention
### 14. Backup Retention (Ansible)
**`ansible/roles/pote/defaults/main.yml`:**
```yaml
pote_backup_retention_days: 90 # 🟢 Adjust based on disk space
```
**Action:** Increase for production (180+ days), decrease for dev (30 days)
### 15. API Keys (`.env` - if you get paid APIs)
**`.env` file:**
```bash
QUIVERQUANT_API_KEY= # 🟢 Optional paid service
FMP_API_KEY= # 🟢 Optional paid service
```
**Action:** Add keys if you subscribe to these services (not required)
---
## 📋 Summary: Quick Action List
### Immediate (Before First Deployment)
1. ✅ Update `.env`: `SMTP_PASSWORD`, `REPORT_RECIPIENTS`, `DATABASE_URL` password
2. ✅ Create `ansible/group_vars/all/vault.yml` with your SSH keys
3. ✅ Encrypt vault: `ansible-vault encrypt ansible/group_vars/all/vault.yml`
4. ✅ Update `ansible/inventory.yml` with your server IPs
5. ✅ Add Gitea secrets: `DB_PASSWORD`, `SMTP_PASSWORD`
### Before Production Use
6. ✅ Verify Git repo URL in `ansible/group_vars/all.yml`
7. ✅ Customize `MARKET_MONITOR_TICKERS` based on research
8. ✅ Adjust cron times for your timezone
9. ✅ Set appropriate log levels per environment
### Optional Enhancements
10. ⭐ Add paid API keys if you subscribe
11. ⭐ Adjust alert sensitivity based on testing
12. ⭐ Customize backup retention per environment
---
## 🔍 How to Find These Files
```bash
# Configuration files:
.env # Main config (not in git)
src/pote/config.py # Python config loader
# Ansible files:
ansible/inventory.yml # Server IPs (copy from .example.yml)
ansible/group_vars/all.yml # Common variables
ansible/group_vars/all/vault.yml # Secrets (create from vault.example.yml)
ansible/group_vars/development.yml # Dev environment
ansible/group_vars/staging.yml # QA environment
ansible/group_vars/production.yml # Prod environment
ansible/roles/pote/defaults/main.yml # All default variables
# Scripts:
scripts/automated_daily_run.sh # Daily automation
scripts/automated_weekly_run.sh # Weekly automation
scripts/setup_cron.sh # Cron setup
scripts/proxmox_setup.sh # Initial server setup
# CI/CD:
.github/workflows/ci.yml # CI pipeline
.github/workflows/deploy.yml # Deployment pipeline
```
---
## 🚨 Security Reminders
1. **NEVER commit `.env` to git** - it's in `.gitignore`
2. **NEVER commit unencrypted `vault.yml`** - always use `ansible-vault encrypt`
3. **NEVER put real passwords in example files** - use placeholders
4. **ALWAYS use strong passwords** - minimum 16 characters, mixed case, numbers, symbols
5. **ALWAYS use Gitea secrets** for CI/CD - never hardcode in workflow files
---
## 📞 Need Help?
- **Ansible Vault:** `ansible-vault --help`
- **Gitea Secrets:** Repository Settings → Secrets → Actions
- **Environment Variables:** See `src/pote/config.py` for all available settings
- **Testing Config:** Run `python scripts/health_check.py` after changes
---
**Last Updated:** December 2025

2
Dockerfile
View File

@ -10,11 +10,11 @@ RUN apt-get update && apt-get install -y \
# Copy project files
COPY pyproject.toml .
COPY README.md .
COPY src/ src/
COPY alembic/ alembic/
COPY alembic.ini .
COPY scripts/ scripts/
# Note: README.md excluded by .dockerignore (not needed in container)
# Install Python dependencies
RUN pip install --no-cache-dir --upgrade pip && \

128
SONARQUBE_QUICKSTART.md
View File

@ -1,128 +0,0 @@
# SonarQube Quick Start
**5-minute setup guide for SonarQube code quality analysis.**
---
## ✅ What's Already Done
- ✅ `sonar-project.properties` - Project configuration
- ✅ CI pipeline job - `sonar-analysis` added
- ✅ Coverage report generation - Integrated with pytest
---
## 🚀 Quick Setup (3 Steps)
### Step 1: Create Project in SonarQube
1. Login to SonarQube: `http://your-server:9000`
2. **Projects** → **Create Project**
3. **Project Key:** `pote`
4. **Display Name:** `POTE`
5. Click **Set Up**
### Step 2: Generate Token
1. **My Account** → **Security** → **Generate Token**
2. **Name:** `POTE CI/CD`
3. **Type:** User Token
4. Click **Generate**
5. **⚠️ COPY THE TOKEN** (you won't see it again!)
### Step 3: Add Secrets to Gitea
1. Go to: `https://git.levkin.ca/ilia/POTE/settings/secrets/actions`
2. Add secret: `SONAR_HOST_URL` = `http://your-server:9000`
3. Add secret: `SONAR_TOKEN` = (paste token from Step 2)
---
## 🧪 Test It
```bash
# Push to dev branch
git push origin dev
# Check CI results
# https://git.levkin.ca/ilia/POTE/actions
# View SonarQube results
# http://your-server:9000/dashboard?id=pote
```
---
## 📋 Configuration
### Project Key
- **Key:** `pote` (in `sonar-project.properties`)
- **Name:** `POTE`
- **Version:** `0.1.0`
### Source Code
- **Sources:** `src/`
- **Tests:** `tests/`
- **Coverage:** `coverage.xml` (auto-generated)
### Exclusions
- `__pycache__/`, `*.pyc`
- `venv/`, `tests/`
- `alembic/versions/`
---
## 🔧 Customize
Edit `sonar-project.properties`:
```properties
sonar.projectKey=pote
sonar.projectName=POTE
sonar.sources=src
sonar.tests=tests
```
---
## 📊 View Results
**SonarQube Dashboard:**
```
http://your-server:9000/dashboard?id=pote
```
**Metrics:**
- Code Coverage
- Bugs & Vulnerabilities
- Code Smells
- Technical Debt
- Quality Gate Status
---
## 🐛 Troubleshooting
### "Project does not exist"
→ Create project manually in SonarQube UI
### "Authentication failed"
→ Check `SONAR_TOKEN` secret is correct
### "Connection refused"
→ Verify `SONAR_HOST_URL` and server accessibility
### "Coverage not found"
→ Ensure pytest runs before SonarScanner (already configured)
---
## 📖 Full Documentation
See: `docs/17_sonarqube_setup.md` for complete guide.
---
**Setup Time:** ~5 minutes
**CI Integration:** ✅ Already done
**Manual Steps:** 3 (create project, generate token, add secrets)

324
TESTING_STATUS.md Normal file
View File

@ -0,0 +1,324 @@
# POTE Testing Status Report
**Date:** December 15, 2025
**Status:** ✅ All Systems Operational - Ready for Deployment
---
## 🎯 Test Suite Summary
### **55 Tests - All Passing ✅**
```
Platform: Python 3.13.5, pytest-9.0.2
Test Duration: ~1.8 seconds
Coverage: ~85% overall
```
### Test Breakdown by Module:
| Module | Tests | Status | Coverage |
|--------|-------|--------|----------|
| **Analytics** | 18 tests | ✅ PASS | 80% |
| **Models** | 7 tests | ✅ PASS | 90% |
| **Ingestion** | 14 tests | ✅ PASS | 85% |
| **Price Loader** | 8 tests | ✅ PASS | 90% |
| **Security Enricher** | 8 tests | ✅ PASS | 85% |
---
## 📊 What's Been Tested?
### ✅ Core Database Operations
- [x] Creating and querying Officials
- [x] Creating and querying Securities
- [x] Creating and querying Trades
- [x] Price data storage and retrieval
- [x] Unique constraints and relationships
- [x] Database migrations (Alembic)
### ✅ Data Ingestion
- [x] House Stock Watcher client (with fixtures)
- [x] Trade loading from JSON
- [x] Security enrichment from yfinance
- [x] Price data fetching and storage
- [x] Idempotent operations (no duplicates)
- [x] Error handling for missing/invalid data
### ✅ Analytics Engine
- [x] Return calculations (buy trades)
- [x] Return calculations (sell trades)
- [x] Multiple time windows (30/60/90/180 days)
- [x] Benchmark comparisons (SPY, QQQ, etc.)
- [x] Abnormal returns (alpha calculations)
- [x] Official performance summaries
- [x] Sector-level analysis
- [x] Disclosure timing analysis
- [x] Top performer rankings
- [x] System-wide statistics
### ✅ Edge Cases
- [x] Missing price data handling
- [x] Trades with no exit price yet
- [x] Sell trades (inverted returns)
- [x] Disclosure lags
- [x] Duplicate prevention
- [x] Invalid date ranges
- [x] Empty result sets
---
## 🧪 Test Types
### 1. Unit Tests (Fast, Isolated)
**Location:** `tests/test_*.py` (excluding integration)
**Purpose:** Test individual functions and classes
**Database:** In-memory SQLite (fresh for each test)
**Speed:** ~0.5 seconds
**Examples:**
- `test_parse_amount_range()` - Parse trade amounts
- `test_normalize_transaction_type()` - Trade type normalization
- `test_get_or_create_security()` - Security deduplication
### 2. Integration Tests (Realistic Scenarios)
**Location:** `tests/test_analytics_integration.py`
**Purpose:** Test complete workflows with synthetic data
**Database:** In-memory SQLite with realistic price data
**Speed:** ~0.7 seconds
**Examples:**
- `test_return_calculation_with_real_data()` - Full return calc pipeline
- `test_benchmark_comparison_with_real_data()` - Alpha calculations
- `test_official_performance_summary()` - Aggregated metrics
**Scenarios Tested:**
- Nancy Pelosi buys NVDA early (strong returns)
- Tommy Tuberville buys NVDA later (good but less alpha)
- 120 days of synthetic price data (realistic trends)
- SPY benchmark comparison
- Multiple time window analysis
---
## 🔧 How to Run Tests Locally
### Quick Test
```bash
cd /home/user/Documents/code/pote
source venv/bin/activate
pytest -v
```
### With Coverage Report
```bash
pytest --cov=src/pote --cov-report=html --cov-report=term
# View: firefox htmlcov/index.html
```
### Specific Test Modules
```bash
# Just analytics
pytest tests/test_analytics.py -v
# Just integration tests
pytest tests/test_analytics_integration.py -v
# Specific test
pytest tests/test_analytics.py::test_return_calculator_basic -v
```
### Watch Mode (Re-run on changes)
```bash
pytest-watch
# or
ptw
```
---
## 🚨 Known Limitations
### 1. External API Dependency
**Issue:** House Stock Watcher API is currently DOWN
**Impact:** Can't fetch live congressional trades automatically
**Workaround:**
- Use fixtures (`scripts/ingest_from_fixtures.py`)
- Manual CSV import (`scripts/scrape_alternative_sources.py`)
- Manual entry (`scripts/add_custom_trades.py`)
### 2. Market Data Limits
**Issue:** yfinance has rate limits and occasional failures
**Impact:** Bulk price fetching may be slow
**Workaround:**
- Fetch in batches
- Add retry logic (already implemented)
- Use caching (already implemented)
### 3. No Live Trading API
**Issue:** We only use public disclosure data (inherent lag)
**Impact:** Trades are 30-45 days delayed by law
**This is expected:** POTE is for research, not real-time trading
---
## 📈 Performance Benchmarks
### Test Execution Time
- **Full suite:** 1.8 seconds
- **Unit tests only:** 0.5 seconds
- **Integration tests:** 0.7 seconds
- **Parallel execution:** ~1.0 second (with pytest-xdist)
### Database Operations
- **Create official:** < 1ms
- **Create trade:** < 1ms
- **Fetch prices (100 days):** ~50ms (in-memory)
- **Calculate returns:** ~10ms per trade
- **Aggregate metrics:** ~50ms for 100 trades
---
## 🎯 Pre-Deployment Checklist
### Before Deploying to Proxmox:
- [x] All tests passing locally
- [x] No linter errors (`make lint`)
- [x] Database migrations work (`alembic upgrade head`)
- [x] Scripts are executable and work
- [x] Environment variables documented
- [x] Sample data available for testing
- [x] Documentation up to date
### On Proxmox Container:
```bash
# 1. Pull latest code
cd ~/pote
git pull
# 2. Update dependencies
pip install -e .
# 3. Run tests
pytest -v
# 4. Run migrations
alembic upgrade head
# 5. Verify system
python ~/status.sh
# 6. Test a script
python scripts/enrich_securities.py
```
---
## 🔄 Continuous Testing
### Git Pre-Commit Hook (Optional)
```bash
#!/bin/bash
# .git/hooks/pre-commit
pytest --tb=short
if [ $? -ne 0 ]; then
echo "Tests failed. Commit aborted."
exit 1
fi
```
### CI/CD Integration (Future)
When you set up GitHub Actions or GitLab CI:
```yaml
# .github/workflows/test.yml
name: Tests
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- run: pip install -e .
- run: pytest -v --cov
```
---
## 📝 Test Maintenance
### Adding New Tests
**When to add tests:**
- Adding new features
- Fixing bugs (write test that fails, then fix)
- Before refactoring (ensure tests pass before & after)
**Where to add tests:**
- Unit tests: `tests/test_<module>.py`
- Integration tests: `tests/test_<feature>_integration.py`
**Example:**
```python
def test_new_feature(test_db_session):
"""Test description."""
session = test_db_session
# Arrange
# Act
# Assert
```
### Updating Fixtures
Fixtures are in `tests/conftest.py`:
- `test_db_session` - Fresh database
- `sample_official` - Test official
- `sample_security` - Test security (AAPL)
- `sample_trade` - Test trade
- `sample_price` - Test price record
---
## 🎉 Summary
### Current Status: **PRODUCTION READY** ✅
**What Works:**
- ✅ All 55 tests passing
- ✅ Full analytics pipeline functional
- ✅ Database operations solid
- ✅ Data ingestion from multiple sources
- ✅ Price fetching from yfinance
- ✅ Security enrichment
- ✅ Return calculations
- ✅ Benchmark comparisons
- ✅ Performance metrics
- ✅ CLI scripts operational
**What's Missing:**
- ❌ Live congressional trade API (external issue - House Stock Watcher down)
- **Workaround:** Manual import, CSV, or alternative APIs available
**Next Steps:**
1. ✅ Tests are complete
2. ✅ Code is ready
3. ➡️ **Deploy to Proxmox** (or continue with Phase 2 features)
4. ➡️ Add more data sources
5. ➡️ Build dashboard (Phase 3)
---
## 📞 Need Help?
See:
- `LOCAL_TEST_GUIDE.md` - Detailed local testing instructions
- `QUICKSTART.md` - Usage guide for deployed system
- `docs/09_data_updates.md` - How to add/update data
- `README.md` - Project overview
**Questions about testing?**
All tests are documented with docstrings - read the test files!

681
docs/14_branch_strategy_and_deployment.md
View File

@ -1,681 +0,0 @@
# Branch Strategy & Multi-Environment Deployment
## Overview
This guide covers setting up a proper Git branching strategy with protected branches for Dev, QA, and Production environments, integrated with your Ansible auto-deployment system.
---
## 🌳 Branch Strategy
### Recommended Branch Structure
```
main (production)
├── qa (quality assurance/staging)
└── dev (development)
```
**Alternative naming:**
```
prod (production)
├── staging (QA)
└── develop (dev)
```
---
## 🔒 Branch Protection Rules
### In Gitea: Repository Settings → Branches
#### 1. `main` (Production) - MOST PROTECTED
**Protection Rules:**
- ✅ **Require pull request reviews** (at least 1 approval)
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Restrict who can push** (only maintainers)
- ✅ **Require signed commits** (optional but recommended)
- ✅ **Block force pushes**
- ✅ **Block deletions**
- ✅ **Require linear history** (no merge commits)
**Merge Strategy:**
- Only merge from `qa` branch
- Require successful QA testing
- Tag releases: `v1.0.0`, `v1.1.0`, etc.
#### 2. `qa` (Staging) - MODERATELY PROTECTED
**Protection Rules:**
- ✅ **Require pull request reviews** (at least 1 approval)
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Block force pushes**
- ✅ **Block deletions**
**Merge Strategy:**
- Merge from `dev` branch
- Run full test suite
- Manual QA testing required
#### 3. `dev` (Development) - LIGHTLY PROTECTED
**Protection Rules:**
- ✅ **Require status checks to pass** (CI must pass)
- ✅ **Block force pushes** (optional)
- ⚠️ Allow direct commits (for rapid development)
**Merge Strategy:**
- Feature branches merge here
- Continuous integration testing
- Auto-deploy to dev environment
---
## 📋 Gitea Branch Protection Setup
### Step-by-Step Configuration
#### 1. Create Branches
```bash
cd /home/user/Documents/code/pote
# Create dev branch from main
git checkout -b dev
git push origin dev
# Create qa branch from main
git checkout -b qa
git push origin qa
# Back to main
git checkout main
```
#### 2. Configure in Gitea UI
```
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
2. Click "Add New Rule"
For MAIN branch:
- Branch name pattern: main
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
- Whitelist: (leave empty or add specific users)
For QA branch:
- Branch name pattern: qa
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
For DEV branch:
- Branch name pattern: dev
- ✅ Require status checks
- ⚠️ Allow direct push (for development)
```
---
## 🚀 Deployment Workflow Integration
### Update Workflows for Multi-Environment
#### 1. Update CI Workflow for All Branches
**File:** `.github/workflows/ci.yml`
```yaml
name: CI
on:
push:
branches: [main, qa, dev]
pull_request:
branches: [main, qa, dev]
jobs:
lint-and-test:
runs-on: ubuntu-latest
# ... existing CI jobs ...
```
#### 2. Create Environment-Specific Deployment Workflows
**File:** `.github/workflows/deploy-dev.yml`
```yaml
name: Deploy to Dev
on:
push:
branches: [dev]
workflow_dispatch:
jobs:
deploy-dev:
runs-on: ubuntu-latest
environment: development
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Deploy to Dev Server
env:
DEV_HOST: ${{ secrets.DEV_HOST }}
DEV_USER: ${{ secrets.DEV_USER }}
DEV_SSH_KEY: ${{ secrets.DEV_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_DEV }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_DEV }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$DEV_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $DEV_HOST >> ~/.ssh/known_hosts
# Deploy
ssh ${DEV_USER}@${DEV_HOST} << 'ENDSSH'
cd ~/pote-dev
git fetch origin
git checkout dev
git pull origin dev
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
```
**File:** `.github/workflows/deploy-qa.yml`
```yaml
name: Deploy to QA
on:
push:
branches: [qa]
workflow_dispatch:
jobs:
deploy-qa:
runs-on: ubuntu-latest
environment: staging
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Deploy to QA Server
env:
QA_HOST: ${{ secrets.QA_HOST }}
QA_USER: ${{ secrets.QA_USER }}
QA_SSH_KEY: ${{ secrets.QA_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD_QA }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_QA }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$QA_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $QA_HOST >> ~/.ssh/known_hosts
# Deploy
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
cd ~/pote-qa
git fetch origin
git checkout qa
git pull origin qa
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
- name: Run Smoke Tests
run: |
ssh ${QA_USER}@${QA_HOST} << 'ENDSSH'
cd ~/pote-qa
source venv/bin/activate
python scripts/health_check.py
ENDSSH
```
**File:** `.github/workflows/deploy-prod.yml`
```yaml
name: Deploy to Production
on:
push:
branches: [main]
workflow_dispatch:
inputs:
confirm:
description: 'Type "DEPLOY" to confirm production deployment'
required: true
jobs:
deploy-prod:
runs-on: ubuntu-latest
environment: production
if: github.event.inputs.confirm == 'DEPLOY' || github.event_name == 'push'
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Create Release Tag
run: |
git tag -a "v$(date +%Y%m%d-%H%M%S)" -m "Production release"
git push origin --tags
- name: Deploy to Production
env:
PROD_HOST: ${{ secrets.PROXMOX_HOST }}
PROD_USER: ${{ secrets.PROXMOX_USER }}
PROD_SSH_KEY: ${{ secrets.PROXMOX_SSH_KEY }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
# Setup SSH
mkdir -p ~/.ssh
echo "$PROD_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H $PROD_HOST >> ~/.ssh/known_hosts
# Backup current production
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
git tag "backup-$(date +%Y%m%d-%H%M%S)"
ENDSSH
# Deploy
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
git fetch origin
git checkout main
git pull origin main
source venv/bin/activate
pip install -e .
alembic upgrade head
ENDSSH
- name: Health Check
run: |
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
source venv/bin/activate
python scripts/health_check.py
ENDSSH
- name: Rollback on Failure
if: failure()
run: |
echo "❌ Deployment failed, rolling back..."
ssh ${PROD_USER}@${PROD_HOST} << 'ENDSSH'
cd ~/pote
latest_backup=$(git tag -l "backup-*" | sort -r | head -1)
git checkout "$latest_backup"
source venv/bin/activate
alembic upgrade head
ENDSSH
```
---
## 🔐 Gitea Secrets for Multi-Environment
### Organize Secrets by Environment
#### Development Secrets
```
DEV_HOST=10.0.10.100
DEV_USER=poteapp
DEV_SSH_KEY=(dev SSH key)
SMTP_PASSWORD_DEV=(dev mail password)
DB_PASSWORD_DEV=dev_password_123
```
#### QA/Staging Secrets
```
QA_HOST=10.0.10.101
QA_USER=poteapp
QA_SSH_KEY=(qa SSH key)
SMTP_PASSWORD_QA=(qa mail password)
DB_PASSWORD_QA=qa_password_123
```
#### Production Secrets
```
PROXMOX_HOST=10.0.10.95
PROXMOX_USER=poteapp
PROXMOX_SSH_KEY=(prod SSH key)
SMTP_PASSWORD=(prod mail password)
DB_PASSWORD=changeme123
```
---
## 📊 Deployment Flow Diagram
```
Developer
│
├─> Commit to feature branch
│
├─> Create PR to dev
│ │
│ ├─> CI runs (tests)
│ │
│ ├─> Merge to dev
│ │
│ └─> Auto-deploy to DEV environment
│
├─> Test in DEV
│
├─> Create PR: dev → qa
│ │
│ ├─> CI runs (tests)
│ │
│ ├─> Code review required
│ │
│ ├─> Merge to qa
│ │
│ └─> Auto-deploy to QA environment
│
├─> QA Testing
│
└─> Create PR: qa → main
│
├─> CI runs (tests)
│
├─> Code review required (2 approvals)
│
├─> Manual approval for prod deploy
│
├─> Merge to main
│
├─> Create release tag
│
└─> Deploy to PRODUCTION
```
---
## 🔄 Integration with Your Ansible System
### Option 1: Gitea Webhooks → Ansible
**Setup:**
1. **In Gitea:** Settings → Webhooks → Add Webhook
- URL: `https://your-ansible-controller/webhook/pote`
- Trigger: Push events
- Branches: `dev`, `qa`, `main`
2. **Ansible Playbook:** `deploy-pote.yml`
```yaml
---
- name: Deploy POTE based on branch
hosts: "{{ target_env }}"
vars:
branch: "{{ git_branch }}"
env: "{{ target_env }}"
tasks:
- name: Pull latest code
git:
repo: gitea@10.0.30.169:ilia/POTE.git
dest: /home/poteapp/pote
version: "{{ branch }}"
force: yes
- name: Install dependencies
pip:
requirements: /home/poteapp/pote/requirements.txt
virtualenv: /home/poteapp/pote/venv
- name: Run migrations
command: alembic upgrade head
args:
chdir: /home/poteapp/pote
environment:
DATABASE_URL: "{{ database_url }}"
- name: Update secrets
template:
src: env.j2
dest: /home/poteapp/pote/.env
mode: 0600
- name: Health check
command: python scripts/health_check.py
args:
chdir: /home/poteapp/pote
```
3. **Ansible Inventory:** `inventory.yml`
```yaml
all:
children:
development:
hosts:
dev-pote:
ansible_host: 10.0.10.100
target_env: development
git_branch: dev
database_url: postgresql://poteuser:dev_pass@localhost/potedb_dev
staging:
hosts:
qa-pote:
ansible_host: 10.0.10.101
target_env: staging
git_branch: qa
database_url: postgresql://poteuser:qa_pass@localhost/potedb_qa
production:
hosts:
prod-pote:
ansible_host: 10.0.10.95
target_env: production
git_branch: main
database_url: postgresql://poteuser:prod_pass@localhost/potedb
```
### Option 2: Gitea Actions → Ansible
**File:** `.github/workflows/ansible-deploy.yml`
```yaml
name: Ansible Deploy
on:
push:
branches: [main, qa, dev]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Determine environment
id: env
run: |
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
echo "environment=production" >> $GITHUB_OUTPUT
echo "host=10.0.10.95" >> $GITHUB_OUTPUT
elif [ "${{ github.ref }}" == "refs/heads/qa" ]; then
echo "environment=staging" >> $GITHUB_OUTPUT
echo "host=10.0.10.101" >> $GITHUB_OUTPUT
else
echo "environment=development" >> $GITHUB_OUTPUT
echo "host=10.0.10.100" >> $GITHUB_OUTPUT
fi
- name: Trigger Ansible
run: |
curl -X POST https://your-ansible-controller/api/deploy \
-H "Authorization: Bearer ${{ secrets.ANSIBLE_TOKEN }}" \
-d '{
"project": "pote",
"environment": "${{ steps.env.outputs.environment }}",
"branch": "${{ github.ref_name }}",
"commit": "${{ github.sha }}"
}'
```
---
## ✅ Complete Setup Checklist
### 1. Git Configuration
- [ ] Create `dev` branch
- [ ] Create `qa` branch
- [ ] Keep `main` branch
- [ ] Push all branches to Gitea
### 2. Gitea Branch Protection
- [ ] Protect `main` (require PR + approval + CI)
- [ ] Protect `qa` (require PR + CI)
- [ ] Configure `dev` (require CI only)
### 3. Gitea Secrets
- [ ] Add DEV environment secrets
- [ ] Add QA environment secrets
- [ ] Add PROD environment secrets
### 4. Workflows
- [ ] Update `ci.yml` for all branches
- [ ] Create `deploy-dev.yml`
- [ ] Create `deploy-qa.yml`
- [ ] Create `deploy-prod.yml`
### 5. Ansible Integration
- [ ] Configure Gitea webhooks (if using webhooks)
- [ ] Update Ansible playbooks for POTE
- [ ] Test deployment to each environment
### 6. Documentation
- [ ] Document deployment process
- [ ] Create runbook for rollbacks
- [ ] Train team on workflow
---
## 🚨 What You're Missing (Important!)
### 1. **Environment Variables per Environment**
Create separate `.env` files:
- `.env.dev`
- `.env.qa`
- `.env.prod`
**Never commit these!** Use Ansible templates or Gitea secrets.
### 2. **Database Migrations Strategy**
```bash
# Test migrations in dev first
alembic upgrade head # dev
# Then qa
alembic upgrade head # qa
# Finally prod (with backup!)
pg_dump potedb > backup.sql
alembic upgrade head # prod
```
### 3. **Rollback Strategy**
```bash
# Git rollback
git checkout <previous-commit>
# Database rollback
alembic downgrade -1
# Or restore from backup
psql potedb < backup.sql
```
### 4. **Monitoring & Alerts**
- Health checks after each deployment
- Email/Slack notifications on failure
- Automated rollback on critical errors
### 5. **Feature Flags**
Consider adding feature flags for gradual rollouts:
```python
# In config.py
FEATURE_NEW_ANALYTICS = os.getenv("FEATURE_NEW_ANALYTICS", "false") == "true"
```
### 6. **Changelog & Release Notes**
Maintain `CHANGELOG.md`:
```markdown
## [1.2.0] - 2025-12-15
### Added
- Email reporting system
- Gitea secrets integration
### Fixed
- Database connection timeout
### Changed
- Improved error handling
```
---
## 📚 Quick Reference Commands
```bash
# Create branches
git checkout -b dev && git push origin dev
git checkout -b qa && git push origin qa
# Merge dev → qa
git checkout qa
git merge dev
git push origin qa
# Merge qa → main (via PR in Gitea!)
# Don't do this directly - use Pull Request
# Check which branch you're on
git branch
# See all branches
git branch -a
# Deploy manually (if Ansible fails)
ssh poteapp@10.0.10.100 "cd ~/pote && git pull origin dev"
```
---
## 🎯 Recommended Next Steps
1. **Right now:** Create dev and qa branches
2. **Today:** Set up branch protection in Gitea
3. **This week:** Create environment-specific workflows
4. **This week:** Integrate with your Ansible system
5. **Next week:** Test full deployment flow
---
**With this setup, you'll have a professional, production-ready deployment pipeline!** 🚀

353
docs/15_branch_setup_checklist.md
View File

@ -1,353 +0,0 @@
# ✅ Branch Strategy Setup Complete!
## 🌳 Branches Created
Your POTE repository now has three branches:
```
✅ main (production) - PROTECTED
✅ qa (staging) - Ready to protect
✅ dev (development) - Ready to protect
```
**Current status:**
- `main` is already protected (you saw the error - that's good!)
- New documentation committed to `dev` branch
- Ready to configure protection for `qa` and `dev`
---
## 🔒 Next Steps: Configure Branch Protection
### Go to Gitea: https://git.levkin.ca/ilia/POTE/settings/branches
### 1. Protect `main` (Production) - Already Done! ✅
Your `main` branch is already protected (we couldn't push directly to it).
**Verify settings:**
- Branch name pattern: `main`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require approvals: 1 (or 2 for production)
- ✅ Require status checks
- ✅ Block force push
- ✅ Block deletion
### 2. Protect `qa` (Staging) - TODO
Click "Add New Rule":
- Branch name pattern: `qa`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ Require status checks to pass
- ✅ Block force push
- ✅ Block deletion
### 3. Configure `dev` (Development) - TODO
Click "Add New Rule":
- Branch name pattern: `dev`
- ✅ Require status checks to pass (CI must pass)
- ⚠️ Allow direct push (for rapid development)
- ✅ Block force push (optional)
---
## 📋 What You're Missing (Checklist)
### ✅ Already Have:
- [x] Three branches (main, qa, dev)
- [x] Main branch protection
- [x] Comprehensive documentation
- [x] CI/CD pipeline
- [x] Gitea secrets integration
### 🔲 Need to Add:
#### 1. **Environment-Specific Secrets in Gitea**
Go to: https://git.levkin.ca/ilia/POTE/settings/secrets
**Development:**
```
DEV_HOST=10.0.10.100 (or your dev server IP)
DEV_USER=poteapp
DEV_SSH_KEY=(SSH key for dev server)
SMTP_PASSWORD_DEV=(dev email password)
DB_PASSWORD_DEV=dev_password_123
```
**QA/Staging:**
```
QA_HOST=10.0.10.101 (or your QA server IP)
QA_USER=poteapp
QA_SSH_KEY=(SSH key for QA server)
SMTP_PASSWORD_QA=(qa email password)
DB_PASSWORD_QA=qa_password_123
```
**Production:**
```
PROXMOX_HOST=10.0.10.95 (already have this)
PROXMOX_USER=poteapp
PROXMOX_SSH_KEY=(already have this)
SMTP_PASSWORD=(already have this)
DB_PASSWORD=changeme123
```
#### 2. **Create Environment-Specific Deployment Workflows**
Files to create:
- `.github/workflows/deploy-dev.yml` (see docs/14_branch_strategy_and_deployment.md)
- `.github/workflows/deploy-qa.yml`
- `.github/workflows/deploy-prod.yml` (already have deploy.yml, can rename/update)
#### 3. **Set Up Separate Servers/Containers**
You need three environments:
| Environment | Server/Container | Database | Purpose |
|-------------|------------------|----------|---------|
| **Dev** | `10.0.10.100` (or new LXC) | `potedb_dev` | Development testing |
| **QA** | `10.0.10.101` (or new LXC) | `potedb_qa` | Pre-production testing |
| **Prod** | `10.0.10.95` (existing) | `potedb` | Production |
**Options:**
- Create 2 more LXC containers (recommended)
- Use same server with different ports/databases
- Use Docker containers
#### 4. **Ansible Integration**
**Option A: Gitea Webhooks**
```
Gitea → Settings → Webhooks → Add Webhook
URL: https://your-ansible-controller/webhook/pote
Trigger on: Push events
Branches: dev, qa, main
```
**Option B: Gitea Actions calls Ansible**
```yaml
# In workflow
- name: Trigger Ansible
run: |
curl -X POST https://ansible-controller/api/deploy \
-d '{"branch": "${{ github.ref_name }}"}'
```
#### 5. **Update Ansible Playbook**
Your Ansible playbook should:
```yaml
- name: Deploy POTE
hosts: "{{ target_env }}"
vars:
branch: "{{ git_branch }}" # dev, qa, or main
tasks:
- git:
repo: gitea@10.0.30.169:ilia/POTE.git
dest: /home/poteapp/pote
version: "{{ branch }}"
# ... rest of deployment
```
#### 6. **Database Migration Strategy**
```bash
# Always test in dev first
ssh poteapp@dev-server "cd ~/pote && alembic upgrade head"
# Then QA
ssh poteapp@qa-server "cd ~/pote && alembic upgrade head"
# Finally prod (with backup!)
ssh poteapp@prod-server "pg_dump potedb > backup.sql && cd ~/pote && alembic upgrade head"
```
#### 7. **Monitoring & Alerts**
Add to each deployment:
```yaml
- name: Health Check
run: python scripts/health_check.py
- name: Send Alert on Failure
if: failure()
run: |
# Send email/Slack notification
```
#### 8. **Environment Variables**
Create separate configs:
- `.env.dev` (in dev server)
- `.env.qa` (in qa server)
- `.env` (in prod server - already have)
**Never commit these!** Use Ansible templates or deployment workflows.
---
## 🚀 Workflow After Setup
### Development Flow:
```bash
# 1. Work on feature
git checkout dev
git pull origin dev
# ... make changes ...
git commit -m "Add feature"
git push origin dev
# 2. Auto-deploys to DEV server
# (via Gitea webhook or Actions)
# 3. Test in DEV
# 4. Promote to QA
# Create PR: dev → qa in Gitea UI
# Merge after approval
# Auto-deploys to QA server
# 5. QA Testing
# 6. Promote to PROD
# Create PR: qa → main in Gitea UI
# Requires 2 approvals
# Merge
# Manual deployment trigger (with confirmation)
```
---
## 📚 Documentation
**Main guide:** `docs/14_branch_strategy_and_deployment.md`
Covers:
- ✅ Branch protection setup
- ✅ Multi-environment workflows
- ✅ Ansible integration
- ✅ Deployment flow
- ✅ Rollback procedures
- ✅ Complete checklist
---
## 🎯 Quick Actions (Do These Now)
### 1. Configure Branch Protection (5 minutes)
```
https://git.levkin.ca/ilia/POTE/settings/branches
- Add rule for 'qa'
- Add rule for 'dev'
```
### 2. Add Environment Secrets (10 minutes)
```
https://git.levkin.ca/ilia/POTE/settings/secrets
- Add DEV_* secrets
- Add QA_* secrets
- Verify PROD secrets exist
```
### 3. Create PR for Documentation (2 minutes)
```
https://git.levkin.ca/ilia/POTE/compare/main...dev
- Create pull request
- Title: "Add branch strategy documentation"
- Merge to main
```
### 4. Decide on Server Setup
**Option 1:** Create 2 more LXC containers
```bash
# On Proxmox host
pct clone 100 101 --hostname pote-dev
pct clone 100 102 --hostname pote-qa
```
**Option 2:** Use existing server with different databases
```bash
# On existing server
createdb potedb_dev
createdb potedb_qa
```
### 5. Configure Ansible
Update your Ansible inventory to include:
- `pote-dev` host
- `pote-qa` host
- `pote-prod` host (existing)
---
## ⚠️ Important Notes
### Main Branch is Protected!
You saw this error:
```
remote: Gitea: Not allowed to push to protected branch main
```
**This is GOOD!** It means:
- ✅ Main branch is protected
- ✅ Can't accidentally push directly
- ✅ Must use Pull Requests
- ✅ Requires code review
**To update main:**
1. Push to `dev` or `qa`
2. Create Pull Request in Gitea
3. Get approval
4. Merge
### Current Branch Status
```bash
$ git branch
dev ← New documentation is here
* main ← Protected, can't push directly
qa ← Empty, same as main
```
---
## 🔗 Links
- **Repository:** https://git.levkin.ca/ilia/POTE
- **Branch Protection:** https://git.levkin.ca/ilia/POTE/settings/branches
- **Secrets:** https://git.levkin.ca/ilia/POTE/settings/secrets
- **Actions:** https://git.levkin.ca/ilia/POTE/actions
- **Create PR:** https://git.levkin.ca/ilia/POTE/compare/main...dev
---
## ✅ Summary
**What's Done:**
- ✅ Created `dev`, `qa`, `main` branches
- ✅ Main branch is protected
- ✅ Documentation committed to `dev`
- ✅ Ready for Ansible integration
**What's Next:**
1. Configure branch protection for `qa` and `dev`
2. Add environment-specific secrets
3. Create PR to merge docs to main
4. Set up dev/qa servers
5. Configure Ansible for multi-environment
6. Test deployment flow
**You're 80% there! Just need to configure Gitea settings and set up the additional servers.** 🚀

378
docs/16_pipeline_setup.md
View File

@ -1,378 +0,0 @@
# 🔧 Pipeline Setup Guide for Branch Protection
## ❓ Do You Need a Pipeline?
**YES!** If you want to use "Require status checks" in branch protection, you need a CI pipeline.
**Good news:** You already have one! ✅
---
## ✅ What You Already Have
### CI Pipeline: `.github/workflows/ci.yml`
**Status:** ✅ Exists and working
**Runs on:** Push to `main`, `qa`, `dev` (just updated!)
**What it does:**
- Runs linters (ruff, black, mypy)
- Runs 93 tests
- Checks code quality
- Uses PostgreSQL for integration tests
---
## 🚀 Setup Order (IMPORTANT!)
### ⚠️ **DO THIS IN ORDER:**
### Step 1: Merge CI Updates to Main (FIRST!)
**Why:** Branch protection needs the CI pipeline to exist in the branch you're protecting.
**How:**
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
2. Click "New Pull Request"
3. Title: "Update CI for multi-branch support"
4. **Merge this PR** (you can merge without protection for now)
**What this does:**
- Updates CI to run on `main`, `qa`, and `dev`
- Makes CI available for branch protection
---
### Step 2: Verify CI is Working
After merging the PR, check:
1. Go to: https://git.levkin.ca/ilia/POTE/actions
2. You should see the CI workflow running
3. Wait for it to complete (green checkmark ✅)
**If CI fails:**
- Don't set up branch protection yet
- Fix the CI issues first
- Ensure tests pass
---
### Step 3: Configure Branch Protection (AFTER CI WORKS)
**Only after CI is passing**, go to:
https://git.levkin.ca/ilia/POTE/settings/branches
#### For `main` Branch (Already Protected)
**Verify these settings:**
- Branch pattern: `main`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1-2 approvals
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test` (this appears after CI runs once)
- ✅ Block force push
- ✅ Block deletion
#### For `qa` Branch
Click "Add New Rule":
- Branch pattern: `qa`
- ✅ Enable push protection
- ✅ Require pull request
- ✅ Require 1 approval
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test`
- ✅ Block force push
- ✅ Block deletion
#### For `dev` Branch
Click "Add New Rule":
- Branch pattern: `dev`
- ✅ **Require status checks to pass before merging**
- Select: `CI / lint-and-test`
- ⚠️ **Allow direct push** (no PR required for dev)
- ⚠️ Allow force push (optional, for rebasing)
---
## 🔍 What "Require Status Checks" Means
When you enable "Require status checks":
**Before merge:**
```
PR created: dev → qa
↓
CI pipeline runs automatically
↓
Tests must pass ✅
↓
Only then can you merge
```
**If CI fails:**
```
PR created: dev → qa
↓
CI pipeline runs
↓
Tests fail ❌
↓
Merge button is DISABLED
↓
Must fix code and push again
```
---
## 📋 Status Checks Available
After your CI runs once, you'll see these options in branch protection:
**Available checks:**
- `CI / lint-and-test` - Main CI pipeline (93 tests)
- `CI / security-scan` - Security scanning
- `CI / dependency-scan` - Dependency vulnerabilities
- `CI / docker-build-test` - Docker build verification
**Recommended:**
- **For `main`:** Require ALL checks ✅
- **For `qa`:** Require `lint-and-test` + `security-scan` ✅
- **For `dev`:** Require `lint-and-test` only ✅
---
## 🎯 Step-by-Step Setup (Complete)
### 1. Merge CI Updates (5 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
2. Create PR: "Update CI for multi-branch support"
3. Merge (you can approve your own PR for now)
4. Wait for CI to run on main branch
```
### 2. Check CI Status (2 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/actions
2. Click on the latest workflow run
3. Verify all jobs pass ✅
```
### 3. Configure Branch Protection (10 minutes)
```
1. Go to: https://git.levkin.ca/ilia/POTE/settings/branches
2. For main (update existing rule):
- ✅ Require status checks
- Select: CI / lint-and-test
3. Add rule for qa:
- Branch: qa
- ✅ Require PR
- ✅ Require status checks
- Select: CI / lint-and-test
4. Add rule for dev:
- Branch: dev
- ✅ Require status checks
- Select: CI / lint-and-test
- ⚠️ Allow direct push
```
---
## ⚠️ Common Issues
### Issue 1: "No status checks found"
**Cause:** CI hasn't run on that branch yet
**Fix:**
```bash
# Push something to trigger CI
git checkout dev
git commit --allow-empty -m "Trigger CI"
git push origin dev
# Wait for CI to run, then configure protection
```
### Issue 2: "Status check never completes"
**Cause:** CI is failing or stuck
**Fix:**
1. Go to Actions tab
2. Check the failing job
3. Fix the issue
4. Push again
### Issue 3: "Can't select status checks in dropdown"
**Cause:** CI workflow name doesn't match
**Fix:**
- Workflow must be named exactly: `CI`
- Job must be named: `lint-and-test`
- Already correct in your `.github/workflows/ci.yml` ✅
---
## 🧪 Test Your Setup
### After configuring protection:
**Test 1: Try to push directly to main**
```bash
git checkout main
git commit --allow-empty -m "Test"
git push origin main
# Should fail: "Not allowed to push to protected branch"
```
**Test 2: Create PR with failing tests**
```bash
git checkout dev
# Break a test intentionally
git commit -m "Break test"
git push origin dev
# Create PR to qa
# Merge button should be disabled until CI passes
```
**Test 3: Create PR with passing tests**
```bash
git checkout dev
# Fix the test
git commit -m "Fix test"
git push origin dev
# Create PR to qa
# Merge button should be enabled after CI passes ✅
```
---
## 📊 What Happens After Setup
### Workflow with Protection:
```
Developer pushes to dev
↓
CI runs automatically
↓
✅ Tests pass
↓
Developer creates PR: dev → qa
↓
CI runs on PR
↓
✅ Tests pass
↓
Reviewer approves
↓
✅ Merge button enabled
↓
Merge to qa
↓
CI runs on qa branch
↓
✅ Tests pass
↓
Auto-deploy to QA server (if configured)
```
**If tests fail at any point:**
```
CI runs
↓
❌ Tests fail
↓
Merge button DISABLED
↓
Developer fixes code
↓
Pushes again
↓
CI runs again
↓
Loop until tests pass ✅
```
---
## ✅ Checklist
Before configuring branch protection:
- [ ] CI workflow exists (`.github/workflows/ci.yml`) ✅
- [ ] CI runs on all branches (`main`, `qa`, `dev`) ✅
- [ ] CI has run at least once on each branch
- [ ] All tests are passing ✅
- [ ] You can see workflow runs in Actions tab
After configuring:
- [ ] `main` branch requires status checks
- [ ] `qa` branch requires status checks
- [ ] `dev` branch requires status checks
- [ ] Tested: Can't push directly to `main`
- [ ] Tested: PR merge blocked when CI fails
- [ ] Tested: PR merge allowed when CI passes
---
## 🎯 Quick Start (TL;DR)
```bash
# 1. Merge CI updates
# Go to: https://git.levkin.ca/ilia/POTE/compare/main...dev
# Create and merge PR
# 2. Wait for CI to run
# Check: https://git.levkin.ca/ilia/POTE/actions
# 3. Configure branch protection
# Go to: https://git.levkin.ca/ilia/POTE/settings/branches
# Add rules for main, qa, dev
# Enable "Require status checks"
# Select "CI / lint-and-test"
# Done! ✅
```
---
## 📚 Related Documentation
- **CI Workflow:** `.github/workflows/ci.yml`
- **Branch Strategy:** `docs/14_branch_strategy_and_deployment.md`
- **Setup Checklist:** `BRANCH_SETUP_COMPLETE.md`
- **Gitea Secrets:** `GITEA_SECRETS_GUIDE.md`
---
## 🚀 Summary
**Do you need a pipeline?**
- ✅ YES, to use "Require status checks"
- ✅ You already have one!
- ✅ Just need to merge it to main first
**Setup order:**
1. Merge CI updates to main (via PR)
2. Verify CI runs and passes
3. Configure branch protection
4. Test the protection
**After setup:**
- All branches protected by CI
- Can't merge failing code
- Professional development workflow
- Ready for Ansible integration
**You're almost there! Just merge the PR and configure protection.** 🎉

414
docs/17_sonarqube_setup.md
View File

@ -1,414 +0,0 @@
# SonarQube Setup Guide for POTE
**Complete guide for setting up SonarQube code quality analysis.**
---
## 🎯 Overview
SonarQube provides:
- **Code Quality Metrics** - Maintainability, reliability, security
- **Technical Debt** - Time to fix issues
- **Code Coverage** - Test coverage visualization
- **Code Smells** - Code quality issues
- **Security Vulnerabilities** - Security hotspots
- **Bug Detection** - Potential bugs
---
## 📋 Prerequisites
### 1. SonarQube Server
- ✅ You mentioned you have the runner (SonarQube server)
- Server URL: `http://your-sonarqube-server:9000` (or your URL)
- Server must be accessible from CI/CD runners
### 2. SonarQube Project
- Project key: `pote` (configured in `sonar-project.properties`)
- Project name: `POTE`
- Can be created manually or auto-created on first scan
### 3. SonarQube Token
- User token with permissions:
- ✅ **Execute Analysis** (required)
- ✅ **Create Projects** (if project doesn't exist)
---
## 🔧 Step 1: Create SonarQube Project
### Option A: Create Manually (Recommended)
1. **Login to SonarQube:**
```
http://your-sonarqube-server:9000
```
2. **Create Project:**
- Go to: **Projects** → **Create Project**
- **Project Key:** `pote`
- **Display Name:** `POTE`
- **Main Branch:** `main`
- Click **Set Up**
3. **Generate Token:**
- Go to: **My Account** → **Security** → **Generate Token**
- **Name:** `POTE CI/CD`
- **Type:** **User Token**
- **Expires:** (set expiration or leave blank)
- Click **Generate**
- **⚠️ COPY THE TOKEN** - You won't see it again!
### Option B: Auto-Create (First Scan)
- Project will be created automatically on first scan
- Token must have **"Create Projects"** permission
---
## 🔐 Step 2: Configure Gitea Secrets
### Add Secrets in Gitea
1. **Go to Repository Settings:**
```
https://git.levkin.ca/ilia/POTE/settings/secrets/actions
```
2. **Add Secret: `SONAR_HOST_URL`**
- **Name:** `SONAR_HOST_URL`
- **Value:** `http://your-sonarqube-server:9000`
- Example: `http://10.0.30.169:9000`
- Click **Add Secret**
3. **Add Secret: `SONAR_TOKEN`**
- **Name:** `SONAR_TOKEN`
- **Value:** (paste the token from Step 1)
- Click **Add Secret**
### Verify Secrets
```bash
# In CI pipeline, secrets are available as:
${{ secrets.SONAR_HOST_URL }}
${{ secrets.SONAR_TOKEN }}
```
---
## 📄 Step 3: Project Configuration
### File: `sonar-project.properties`
Already created in project root with these settings:
```properties
# Project identification
sonar.projectKey=pote
sonar.projectName=POTE
sonar.projectVersion=0.1.0
# Source code location
sonar.sources=src
sonar.sourceEncoding=UTF-8
# Test code location
sonar.tests=tests
sonar.test.inclusions=**/test_*.py
# Exclusions
sonar.exclusions=**/__pycache__/**,**/*.pyc,**/venv/**,**/tests/**,**/alembic/versions/**
# Python-specific settings
sonar.python.version=3.11
# Coverage reports
sonar.python.coverage.reportPaths=coverage.xml
```
### Customize if Needed
Edit `sonar-project.properties` to:
- Change project key/name
- Add more exclusions
- Configure additional settings
---
## 🚀 Step 4: CI Pipeline Integration
### Already Configured!
The CI pipeline (`.github/workflows/ci.yml`) includes:
1. **SonarScanner Installation**
- Downloads and installs SonarScanner CLI
- Version: `5.0.1.3006` (stable)
2. **Coverage Report Generation**
- Runs pytest with coverage
- Generates `coverage.xml` for SonarQube
3. **SonarScanner Execution**
- Runs analysis
- Uploads results to SonarQube server
- Non-blocking (won't fail CI if SonarQube is unavailable)
### Job: `sonar-analysis`
```yaml
sonar-analysis:
runs-on: ubuntu-latest
env:
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
steps:
- Install SonarScanner
- Generate coverage report
- Run SonarScanner analysis
```
---
## 🧪 Step 5: Test Locally (Optional)
### Install SonarScanner Locally
```bash
# Download SonarScanner
wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-5.0.1.3006-linux.zip
unzip sonar-scanner-cli-5.0.1.3006-linux.zip
export PATH=$PATH:$(pwd)/sonar-scanner-5.0.1.3006-linux/bin
```
### Run Analysis Locally
```bash
cd /home/user/Documents/code/pote
# Generate coverage report
source venv/bin/activate
pytest tests/ --cov=src/pote --cov-report=xml
# Run SonarScanner
sonar-scanner \
-Dsonar.host.url=http://your-sonarqube-server:9000 \
-Dsonar.token=your_token_here
```
### Or Use Environment Variables
```bash
export SONAR_HOST_URL=http://your-sonarqube-server:9000
export SONAR_TOKEN=your_token_here
sonar-scanner
```
---
## 📊 Step 6: View Results
### In SonarQube UI
1. **Go to Project:**
```
http://your-sonarqube-server:9000/dashboard?id=pote
```
2. **View Metrics:**
- **Overview** - Overall quality gate status
- **Issues** - Bugs, vulnerabilities, code smells
- **Measures** - Coverage, complexity, duplications
- **Code** - Source code with inline issues
- **Activity** - Analysis history
### Quality Gate
SonarQube will show:
- ✅ **Pass** - Meets quality standards
- ❌ **Fail** - Doesn't meet quality standards
### Common Metrics
- **Coverage** - Test coverage percentage
- **Duplications** - Code duplication percentage
- **Maintainability Rating** - A-E rating
- **Reliability Rating** - A-E rating
- **Security Rating** - A-E rating
- **Technical Debt** - Time to fix all issues
---
## 🔧 Configuration Options
### Quality Gate
Configure in SonarQube:
- **Projects** → **POTE** → **Quality Gates**
- Set thresholds for:
- Coverage
- Duplications
- Bugs
- Vulnerabilities
- Code smells
### Exclusions
Add to `sonar-project.properties`:
```properties
# Exclude specific files
sonar.exclusions=**/legacy/**,**/old_code/**
# Exclude specific issues
sonar.issue.ignore.multicriteria=e1
sonar.issue.ignore.multicriteria.e1.ruleKey=python:S1234
sonar.issue.ignore.multicriteria.e1.resourceKey=**/*.py
```
### Coverage Exclusions
```properties
# Exclude from coverage
sonar.coverage.exclusions=**/tests/**,**/migrations/**,**/__init__.py
```
---
## 🐛 Troubleshooting
### Issue: "Project 'pote' does not exist"
**Solution:**
1. Create project manually in SonarQube UI
2. OR ensure token has "Create Projects" permission
### Issue: "Authentication failed"
**Solution:**
1. Verify `SONAR_TOKEN` secret is correct
2. Check token hasn't expired
3. Regenerate token if needed
### Issue: "Connection refused"
**Solution:**
1. Verify `SONAR_HOST_URL` is correct
2. Check SonarQube server is running
3. Verify network connectivity from CI runner
4. Check firewall rules
### Issue: "Coverage report not found"
**Solution:**
1. Ensure pytest runs before SonarScanner
2. Check `coverage.xml` is generated
3. Verify path in `sonar-project.properties`:
```properties
sonar.python.coverage.reportPaths=coverage.xml
```
### Issue: "No files to analyze"
**Solution:**
1. Check `sonar.sources=src` is correct
2. Verify source files aren't excluded
3. Check file encoding (should be UTF-8)
---
## 📈 Best Practices
### 1. Regular Analysis
- Run on every commit (already configured in CI)
- Review results regularly
- Fix issues incrementally
### 2. Quality Gate
- Set realistic thresholds
- Start with warnings, not failures
- Gradually increase standards
### 3. Coverage
- Aim for 80%+ coverage
- Focus on critical paths first
- Don't sacrifice quality for coverage
### 4. Technical Debt
- Track and reduce over time
- Prioritize high-impact issues
- Set time limits for fixing
### 5. Team Integration
- Share SonarQube dashboard with team
- Review issues in code reviews
- Use quality gate in PR checks
---
## 🔗 Integration with CI/CD
### Current Setup
✅ **Already Integrated:**
- Runs automatically on every push
- Non-blocking (won't fail CI)
- Generates coverage report
- Uploads results to SonarQube
### Make Quality Gate Blocking (Optional)
To fail CI if quality gate fails:
```yaml
- name: Run SonarScanner
run: |
sonar-scanner \
-Dsonar.qualitygate.wait=true \
...
```
Then remove `continue-on-error: true` from the step.
---
## 📝 Summary
### Quick Setup Checklist
- [ ] SonarQube server running and accessible
- [ ] Project `pote` created in SonarQube (or auto-create enabled)
- [ ] Token generated with "Execute Analysis" permission
- [ ] `SONAR_HOST_URL` secret added to Gitea
- [ ] `SONAR_TOKEN` secret added to Gitea
- [ ] `sonar-project.properties` file exists (✅ already created)
- [ ] CI pipeline includes `sonar-analysis` job (✅ already added)
- [ ] Test by pushing to dev branch
### Files Created
- ✅ `sonar-project.properties` - Project configuration
- ✅ `.github/workflows/ci.yml` - Updated with SonarScanner job
### Next Steps
1. **Add secrets to Gitea** (Step 2)
2. **Push to dev branch** - CI will run SonarScanner
3. **View results** in SonarQube dashboard
4. **Configure quality gate** as needed
5. **Review and fix issues** incrementally
---
## 📞 Support
- **SonarQube Docs:** https://docs.sonarqube.org/
- **SonarScanner Docs:** https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/
- **Python Plugin:** https://docs.sonarqube.org/latest/analysis/languages/python/
---
**Last Updated:** December 2025
**SonarScanner Version:** 5.0.1.3006
**Python Version:** 3.11

1
pyproject.toml
View File

@ -18,7 +18,6 @@ dependencies = [
"pydantic-settings>=2.0",
"python-dotenv>=1.0",
"requests>=2.31",
"httpx>=0.24",
"pandas>=2.0",
"numpy>=1.24",
"yfinance>=0.2",

40
sonar-project.properties
View File

@ -1,40 +0,0 @@
# SonarQube Project Configuration for POTE
# ===========================================
# Project identification
sonar.projectKey=pote
sonar.projectName=POTE
sonar.projectVersion=0.1.0
# Source code location
sonar.sources=src
sonar.sourceEncoding=UTF-8
# Test code location
sonar.tests=tests
sonar.test.inclusions=**/test_*.py
# Exclusions
sonar.exclusions=**/__pycache__/**,**/*.pyc,**/venv/**,**/tests/**,**/alembic/versions/**,**/*.egg-info/**,**/pote.egg-info/**
# Python-specific settings
sonar.python.version=3.11
# Coverage reports (generated by pytest-cov)
sonar.python.coverage.reportPaths=coverage.xml
# Code analysis
sonar.python.xunit.reportPath=test-results.xml
# Language
sonar.language=py
# Encoding
sonar.sourceEncoding=UTF-8
# SCM (optional - if you want to track blame)
# sonar.scm.provider=git
# Quality Gate
# sonar.qualitygate.wait=true