Features Added: ============== 📧 EMAIL REPORTING SYSTEM: - EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom) - ReportGenerator: Generate daily/weekly summaries with HTML/text formatting - Configurable via .env (SMTP_HOST, SMTP_PORT, etc.) - Scripts: send_daily_report.py, send_weekly_report.py 🤖 AUTOMATED RUNS: - automated_daily_run.sh: Full daily ETL pipeline + reporting - automated_weekly_run.sh: Weekly pattern analysis + reports - setup_cron.sh: Interactive cron job setup (5-minute setup) - Logs saved to ~/logs/ with automatic cleanup 🔍 HEALTH CHECKS: - health_check.py: System health monitoring - Checks: DB connection, data freshness, counts, recent alerts - JSON output for programmatic use - Exit codes for monitoring integration 🚀 CI/CD PIPELINE: - .github/workflows/ci.yml: Full CI/CD pipeline - GitHub Actions / Gitea Actions compatible - Jobs: lint & test, security scan, dependency scan, Docker build - PostgreSQL service for integration tests - 93 tests passing in CI 📚 COMPREHENSIVE DOCUMENTATION: - AUTOMATION_QUICKSTART.md: 5-minute email setup guide - docs/12_automation_and_reporting.md: Full automation guide - Updated README.md with automation links - Deployment → Production workflow guide 🛠️ IMPROVEMENTS: - All shell scripts made executable - Environment variable examples in .env.example - Report logs saved with timestamps - 30-day log retention with auto-cleanup - Health checks can be scheduled via cron WHAT THIS ENABLES: ================== After deployment, users can: 1. Set up automated daily/weekly email reports (5 min) 2. Receive HTML+text emails with: - New trades, market alerts, suspicious timing - Weekly patterns, rankings, repeat offenders 3. Monitor system health automatically 4. Run full CI/CD pipeline on every commit 5. Deploy with confidence (tests + security scans) USAGE: ====== # One-time setup (on deployed server) ./scripts/setup_cron.sh # Or manually send reports python scripts/send_daily_report.py --to user@example.com python scripts/send_weekly_report.py --to user@example.com # Check system health python scripts/health_check.py See AUTOMATION_QUICKSTART.md for full instructions. 93 tests passing | Full CI/CD | Email reports ready
152 lines
3.9 KiB
Bash
Executable File
152 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Setup Automation for POTE
|
|
# Run this once on your Proxmox container to enable daily updates
|
|
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo " POTE Automation Setup"
|
|
echo "=========================================="
|
|
|
|
# Detect if we're root or regular user
|
|
if [ "$EUID" -eq 0 ]; then
|
|
echo "⚠️ Running as root. Will setup for poteapp user."
|
|
TARGET_USER="poteapp"
|
|
TARGET_HOME="/home/poteapp"
|
|
else
|
|
TARGET_USER="$USER"
|
|
TARGET_HOME="$HOME"
|
|
fi
|
|
|
|
POTE_DIR="${TARGET_HOME}/pote"
|
|
|
|
# Check if POTE directory exists
|
|
if [ ! -d "$POTE_DIR" ]; then
|
|
echo "❌ Error: POTE directory not found at $POTE_DIR"
|
|
echo " Please clone the repository first."
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Found POTE at: $POTE_DIR"
|
|
|
|
# Make scripts executable
|
|
echo ""
|
|
echo "Making scripts executable..."
|
|
chmod +x "${POTE_DIR}/scripts/daily_fetch.sh"
|
|
chmod +x "${POTE_DIR}/scripts/fetch_congressional_trades.py"
|
|
chmod +x "${POTE_DIR}/scripts/enrich_securities.py"
|
|
chmod +x "${POTE_DIR}/scripts/fetch_sample_prices.py"
|
|
|
|
# Create logs directory
|
|
echo "Creating logs directory..."
|
|
mkdir -p "${POTE_DIR}/logs"
|
|
|
|
# Test the daily fetch script
|
|
echo ""
|
|
echo "Testing daily fetch script (dry run)..."
|
|
echo "This may take a few minutes..."
|
|
cd "$POTE_DIR"
|
|
|
|
if [ "$EUID" -eq 0 ]; then
|
|
su - $TARGET_USER -c "cd ${POTE_DIR} && source venv/bin/activate && python --version"
|
|
else
|
|
source venv/bin/activate
|
|
python --version
|
|
fi
|
|
|
|
# Setup cron job
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Cron Job Setup"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Choose schedule:"
|
|
echo " 1) Daily at 7 AM (recommended)"
|
|
echo " 2) Twice daily (7 AM and 7 PM)"
|
|
echo " 3) Weekdays only at 7 AM"
|
|
echo " 4) Custom (I'll help you configure)"
|
|
echo " 5) Skip (manual setup)"
|
|
echo ""
|
|
read -p "Enter choice [1-5]: " choice
|
|
|
|
CRON_LINE=""
|
|
|
|
case $choice in
|
|
1)
|
|
CRON_LINE="0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
2)
|
|
CRON_LINE="0 7,19 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
3)
|
|
CRON_LINE="0 7 * * 1-5 ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
4)
|
|
echo ""
|
|
echo "Cron format: MIN HOUR DAY MONTH WEEKDAY"
|
|
echo "Examples:"
|
|
echo " 0 7 * * * = Daily at 7 AM"
|
|
echo " 0 */6 * * * = Every 6 hours"
|
|
echo " 0 0 * * 0 = Weekly on Sunday"
|
|
read -p "Enter cron schedule: " custom_schedule
|
|
CRON_LINE="${custom_schedule} ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
5)
|
|
echo "Skipping cron setup. You can add manually with:"
|
|
echo " crontab -e"
|
|
echo " Add: 0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
CRON_LINE=""
|
|
;;
|
|
*)
|
|
echo "Invalid choice. Skipping cron setup."
|
|
CRON_LINE=""
|
|
;;
|
|
esac
|
|
|
|
if [ -n "$CRON_LINE" ]; then
|
|
echo ""
|
|
echo "Adding to crontab: $CRON_LINE"
|
|
|
|
if [ "$EUID" -eq 0 ]; then
|
|
# Add as target user
|
|
(su - $TARGET_USER -c "crontab -l" 2>/dev/null || true; echo "$CRON_LINE") | \
|
|
su - $TARGET_USER -c "crontab -"
|
|
else
|
|
# Add as current user
|
|
(crontab -l 2>/dev/null || true; echo "$CRON_LINE") | crontab -
|
|
fi
|
|
|
|
echo "✅ Cron job added!"
|
|
echo ""
|
|
echo "View with: crontab -l"
|
|
fi
|
|
|
|
# Summary
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Setup Complete!"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "📝 What was configured:"
|
|
echo " ✅ Scripts made executable"
|
|
echo " ✅ Logs directory created: ${POTE_DIR}/logs"
|
|
if [ -n "$CRON_LINE" ]; then
|
|
echo " ✅ Cron job scheduled"
|
|
fi
|
|
echo ""
|
|
echo "🧪 Test manually:"
|
|
echo " ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
echo ""
|
|
echo "📊 View logs:"
|
|
echo " tail -f ${POTE_DIR}/logs/daily_fetch_\$(date +%Y%m%d).log"
|
|
echo ""
|
|
echo "⚙️ Manage cron:"
|
|
echo " crontab -l # View cron jobs"
|
|
echo " crontab -e # Edit cron jobs"
|
|
echo ""
|
|
echo "📚 Documentation:"
|
|
echo " ${POTE_DIR}/docs/10_automation.md"
|
|
echo ""
|
|
|
|
|