Features Added: ============== 📧 EMAIL REPORTING SYSTEM: - EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom) - ReportGenerator: Generate daily/weekly summaries with HTML/text formatting - Configurable via .env (SMTP_HOST, SMTP_PORT, etc.) - Scripts: send_daily_report.py, send_weekly_report.py 🤖 AUTOMATED RUNS: - automated_daily_run.sh: Full daily ETL pipeline + reporting - automated_weekly_run.sh: Weekly pattern analysis + reports - setup_cron.sh: Interactive cron job setup (5-minute setup) - Logs saved to ~/logs/ with automatic cleanup 🔍 HEALTH CHECKS: - health_check.py: System health monitoring - Checks: DB connection, data freshness, counts, recent alerts - JSON output for programmatic use - Exit codes for monitoring integration 🚀 CI/CD PIPELINE: - .github/workflows/ci.yml: Full CI/CD pipeline - GitHub Actions / Gitea Actions compatible - Jobs: lint & test, security scan, dependency scan, Docker build - PostgreSQL service for integration tests - 93 tests passing in CI 📚 COMPREHENSIVE DOCUMENTATION: - AUTOMATION_QUICKSTART.md: 5-minute email setup guide - docs/12_automation_and_reporting.md: Full automation guide - Updated README.md with automation links - Deployment → Production workflow guide 🛠️ IMPROVEMENTS: - All shell scripts made executable - Environment variable examples in .env.example - Report logs saved with timestamps - 30-day log retention with auto-cleanup - Health checks can be scheduled via cron WHAT THIS ENABLES: ================== After deployment, users can: 1. Set up automated daily/weekly email reports (5 min) 2. Receive HTML+text emails with: - New trades, market alerts, suspicious timing - Weekly patterns, rankings, repeat offenders 3. Monitor system health automatically 4. Run full CI/CD pipeline on every commit 5. Deploy with confidence (tests + security scans) USAGE: ====== # One-time setup (on deployed server) ./scripts/setup_cron.sh # Or manually send reports python scripts/send_daily_report.py --to user@example.com python scripts/send_weekly_report.py --to user@example.com # Check system health python scripts/health_check.py See AUTOMATION_QUICKSTART.md for full instructions. 93 tests passing | Full CI/CD | Email reports ready
110 lines
3.2 KiB
Bash
Executable File
110 lines
3.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# POTE Automated Daily Run
|
|
# This script should be run by cron daily (e.g., at 6 AM after market close)
|
|
#
|
|
# Example crontab entry:
|
|
# 0 6 * * * /home/poteapp/pote/scripts/automated_daily_run.sh >> /home/poteapp/logs/daily_run.log 2>&1
|
|
|
|
set -e
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
|
LOG_DIR="${LOG_DIR:-$HOME/logs}"
|
|
VENV_PATH="${VENV_PATH:-$PROJECT_ROOT/venv}"
|
|
REPORT_RECIPIENTS="${REPORT_RECIPIENTS:-admin@localhost}"
|
|
|
|
# Create log directory if it doesn't exist
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Timestamp for logging
|
|
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo "==============================================="
|
|
echo "POTE Automated Daily Run - $TIMESTAMP"
|
|
echo "==============================================="
|
|
|
|
# Activate virtual environment
|
|
if [ -d "$VENV_PATH" ]; then
|
|
echo "Activating virtual environment..."
|
|
source "$VENV_PATH/bin/activate"
|
|
else
|
|
echo "WARNING: Virtual environment not found at $VENV_PATH"
|
|
echo "Attempting to use system Python..."
|
|
fi
|
|
|
|
# Change to project directory
|
|
cd "$PROJECT_ROOT"
|
|
|
|
# Load environment variables
|
|
if [ -f ".env" ]; then
|
|
echo "Loading environment variables from .env..."
|
|
export $(grep -v '^#' .env | xargs)
|
|
fi
|
|
|
|
# Step 1: Fetch new congressional trades
|
|
echo ""
|
|
echo "[1/6] Fetching congressional trades..."
|
|
if python scripts/fetch_congressional_trades.py; then
|
|
echo "✓ Congressional trades fetched successfully"
|
|
else
|
|
echo "⚠ Warning: Failed to fetch congressional trades (may be API issue)"
|
|
fi
|
|
|
|
# Step 2: Enrich securities (get company names, sectors)
|
|
echo ""
|
|
echo "[2/6] Enriching security data..."
|
|
if python scripts/enrich_securities.py; then
|
|
echo "✓ Securities enriched successfully"
|
|
else
|
|
echo "⚠ Warning: Failed to enrich securities"
|
|
fi
|
|
|
|
# Step 3: Fetch latest price data
|
|
echo ""
|
|
echo "[3/6] Fetching price data..."
|
|
if python scripts/fetch_sample_prices.py; then
|
|
echo "✓ Price data fetched successfully"
|
|
else
|
|
echo "⚠ Warning: Failed to fetch price data"
|
|
fi
|
|
|
|
# Step 4: Run market monitoring
|
|
echo ""
|
|
echo "[4/6] Running market monitoring..."
|
|
if python scripts/monitor_market.py --scan; then
|
|
echo "✓ Market monitoring completed"
|
|
else
|
|
echo "⚠ Warning: Market monitoring failed"
|
|
fi
|
|
|
|
# Step 5: Analyze disclosure timing
|
|
echo ""
|
|
echo "[5/6] Analyzing disclosure timing..."
|
|
if python scripts/analyze_disclosure_timing.py --recent 7 --save /tmp/pote_timing_analysis.txt; then
|
|
echo "✓ Disclosure timing analysis completed"
|
|
else
|
|
echo "⚠ Warning: Disclosure timing analysis failed"
|
|
fi
|
|
|
|
# Step 6: Send daily report
|
|
echo ""
|
|
echo "[6/6] Sending daily report..."
|
|
if python scripts/send_daily_report.py --to "$REPORT_RECIPIENTS" --save-to-file "$LOG_DIR/daily_report_$(date +%Y%m%d).txt"; then
|
|
echo "✓ Daily report sent successfully"
|
|
else
|
|
echo "✗ ERROR: Failed to send daily report"
|
|
exit 1
|
|
fi
|
|
|
|
# Final summary
|
|
echo ""
|
|
echo "==============================================="
|
|
echo "Daily run completed successfully at $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "==============================================="
|
|
|
|
# Clean up old log files (keep last 30 days)
|
|
find "$LOG_DIR" -name "daily_report_*.txt" -mtime +30 -delete 2>/dev/null || true
|
|
|
|
exit 0
|
|
|