Features Added: ============== 📧 EMAIL REPORTING SYSTEM: - EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom) - ReportGenerator: Generate daily/weekly summaries with HTML/text formatting - Configurable via .env (SMTP_HOST, SMTP_PORT, etc.) - Scripts: send_daily_report.py, send_weekly_report.py 🤖 AUTOMATED RUNS: - automated_daily_run.sh: Full daily ETL pipeline + reporting - automated_weekly_run.sh: Weekly pattern analysis + reports - setup_cron.sh: Interactive cron job setup (5-minute setup) - Logs saved to ~/logs/ with automatic cleanup 🔍 HEALTH CHECKS: - health_check.py: System health monitoring - Checks: DB connection, data freshness, counts, recent alerts - JSON output for programmatic use - Exit codes for monitoring integration 🚀 CI/CD PIPELINE: - .github/workflows/ci.yml: Full CI/CD pipeline - GitHub Actions / Gitea Actions compatible - Jobs: lint & test, security scan, dependency scan, Docker build - PostgreSQL service for integration tests - 93 tests passing in CI 📚 COMPREHENSIVE DOCUMENTATION: - AUTOMATION_QUICKSTART.md: 5-minute email setup guide - docs/12_automation_and_reporting.md: Full automation guide - Updated README.md with automation links - Deployment → Production workflow guide 🛠️ IMPROVEMENTS: - All shell scripts made executable - Environment variable examples in .env.example - Report logs saved with timestamps - 30-day log retention with auto-cleanup - Health checks can be scheduled via cron WHAT THIS ENABLES: ================== After deployment, users can: 1. Set up automated daily/weekly email reports (5 min) 2. Receive HTML+text emails with: - New trades, market alerts, suspicious timing - Weekly patterns, rankings, repeat offenders 3. Monitor system health automatically 4. Run full CI/CD pipeline on every commit 5. Deploy with confidence (tests + security scans) USAGE: ====== # One-time setup (on deployed server) ./scripts/setup_cron.sh # Or manually send reports python scripts/send_daily_report.py --to user@example.com python scripts/send_weekly_report.py --to user@example.com # Check system health python scripts/health_check.py See AUTOMATION_QUICKSTART.md for full instructions. 93 tests passing | Full CI/CD | Email reports ready
118 lines
3.9 KiB
Python
Executable File
118 lines
3.9 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""
|
|
Real-time market monitoring for congressional tickers.
|
|
Run this continuously or on a schedule to detect unusual activity.
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import click
|
|
|
|
from pote.db import get_session
|
|
from pote.monitoring.alert_manager import AlertManager
|
|
from pote.monitoring.market_monitor import MarketMonitor
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@click.command()
|
|
@click.option("--tickers", help="Comma-separated list of tickers (default: congressional watchlist)")
|
|
@click.option("--interval", default=300, help="Scan interval in seconds (default: 300 = 5 minutes)")
|
|
@click.option("--once", is_flag=True, help="Run once and exit (no continuous monitoring)")
|
|
@click.option("--min-severity", default=5, help="Minimum severity to report (1-10)")
|
|
@click.option("--save-report", help="Save report to file")
|
|
@click.option("--lookback", default=5, help="Days of history to analyze (default: 5)")
|
|
def main(tickers, interval, once, min_severity, save_report, lookback):
|
|
"""Monitor market for unusual activity in congressional tickers."""
|
|
|
|
session = next(get_session())
|
|
monitor = MarketMonitor(session)
|
|
alert_mgr = AlertManager(session)
|
|
|
|
# Parse tickers if provided
|
|
ticker_list = None
|
|
if tickers:
|
|
ticker_list = [t.strip().upper() for t in tickers.split(",")]
|
|
logger.info(f"Monitoring {len(ticker_list)} specified tickers")
|
|
else:
|
|
logger.info("Monitoring congressional watchlist")
|
|
|
|
def run_scan():
|
|
"""Run a single scan."""
|
|
logger.info("=" * 80)
|
|
logger.info(f"Starting market scan at {datetime.now()}")
|
|
logger.info("=" * 80)
|
|
|
|
try:
|
|
# Scan for unusual activity
|
|
alerts = monitor.scan_watchlist(tickers=ticker_list, lookback_days=lookback)
|
|
|
|
if alerts:
|
|
logger.info(f"\n🔔 Found {len(alerts)} alerts!")
|
|
|
|
# Save to database
|
|
monitor.save_alerts(alerts)
|
|
|
|
# Get MarketAlert objects for reporting
|
|
from pote.db.models import MarketAlert
|
|
|
|
alert_objects = (
|
|
session.query(MarketAlert)
|
|
.order_by(MarketAlert.timestamp.desc())
|
|
.limit(len(alerts))
|
|
.all()
|
|
)
|
|
|
|
# Filter by severity
|
|
filtered = alert_mgr.filter_alerts(alert_objects, min_severity=min_severity)
|
|
|
|
if filtered:
|
|
# Generate report
|
|
report = alert_mgr.generate_summary_report(filtered, format="text")
|
|
print("\n" + report)
|
|
|
|
# Save report if requested
|
|
if save_report:
|
|
Path(save_report).write_text(report)
|
|
logger.info(f"Report saved to {save_report}")
|
|
else:
|
|
logger.info(f"No alerts above severity {min_severity}")
|
|
else:
|
|
logger.info("✅ No unusual activity detected")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error during scan: {e}", exc_info=True)
|
|
|
|
logger.info("=" * 80)
|
|
logger.info(f"Scan complete at {datetime.now()}")
|
|
logger.info("=" * 80)
|
|
|
|
# Run scan
|
|
run_scan()
|
|
|
|
# Continuous monitoring mode
|
|
if not once:
|
|
logger.info(f"\n🔄 Continuous monitoring enabled (interval: {interval}s)")
|
|
logger.info("Press Ctrl+C to stop\n")
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(interval)
|
|
run_scan()
|
|
except KeyboardInterrupt:
|
|
logger.info("\n\n⏹️ Monitoring stopped by user")
|
|
else:
|
|
logger.info("\n✅ Single scan complete (use --interval for continuous monitoring)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
|