Add complete automation, reporting, and CI/CD system

Features Added:
==============

📧 EMAIL REPORTING SYSTEM:
- EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom)
- ReportGenerator: Generate daily/weekly summaries with HTML/text formatting
- Configurable via .env (SMTP_HOST, SMTP_PORT, etc.)
- Scripts: send_daily_report.py, send_weekly_report.py

🤖 AUTOMATED RUNS:
- automated_daily_run.sh: Full daily ETL pipeline + reporting
- automated_weekly_run.sh: Weekly pattern analysis + reports
- setup_cron.sh: Interactive cron job setup (5-minute setup)
- Logs saved to ~/logs/ with automatic cleanup

🔍 HEALTH CHECKS:
- health_check.py: System health monitoring
- Checks: DB connection, data freshness, counts, recent alerts
- JSON output for programmatic use
- Exit codes for monitoring integration

🚀 CI/CD PIPELINE:
- .github/workflows/ci.yml: Full CI/CD pipeline
- GitHub Actions / Gitea Actions compatible
- Jobs: lint & test, security scan, dependency scan, Docker build
- PostgreSQL service for integration tests
- 93 tests passing in CI

📚 COMPREHENSIVE DOCUMENTATION:
- AUTOMATION_QUICKSTART.md: 5-minute email setup guide
- docs/12_automation_and_reporting.md: Full automation guide
- Updated README.md with automation links
- Deployment → Production workflow guide

🛠️ IMPROVEMENTS:
- All shell scripts made executable
- Environment variable examples in .env.example
- Report logs saved with timestamps
- 30-day log retention with auto-cleanup
- Health checks can be scheduled via cron

WHAT THIS ENABLES:
==================
After deployment, users can:
1. Set up automated daily/weekly email reports (5 min)
2. Receive HTML+text emails with:
   - New trades, market alerts, suspicious timing
   - Weekly patterns, rankings, repeat offenders
3. Monitor system health automatically
4. Run full CI/CD pipeline on every commit
5. Deploy with confidence (tests + security scans)

USAGE:
======
# One-time setup (on deployed server)
./scripts/setup_cron.sh

# Or manually send reports
python scripts/send_daily_report.py --to user@example.com
python scripts/send_weekly_report.py --to user@example.com

# Check system health
python scripts/health_check.py

See AUTOMATION_QUICKSTART.md for full instructions.

93 tests passing | Full CI/CD | Email reports ready
This commit is contained in:
ilia
2025-12-15 15:34:31 -05:00
parent 53d631a903
commit 0d8d85adc1
44 changed files with 2206 additions and 61 deletions
+1
View File
@@ -12,3 +12,4 @@ __all__ = [
"PerformanceMetrics",
]
+1
View File
@@ -220,3 +220,4 @@ class BenchmarkComparison:
"window_days": window_days,
}
+1
View File
@@ -289,3 +289,4 @@ class PerformanceMetrics:
**aggregate,
}
+1
View File
@@ -242,3 +242,4 @@ class AlertManager:
html_parts.append("</body></html>")
return "\n".join(html_parts)
@@ -356,3 +356,4 @@ class DisclosureCorrelator:
"analyses": sorted(analyses, key=lambda x: x["timing_score"], reverse=True),
}
+1
View File
@@ -279,3 +279,4 @@ class MarketMonitor:
return summary
+1
View File
@@ -357,3 +357,4 @@ class PatternDetector:
"party_comparison": party_comparison,
}
+12
View File
@@ -0,0 +1,12 @@
"""
POTE Reporting Module
Generates and sends formatted reports via email, files, or other channels.
"""
from .email_reporter import EmailReporter
from .report_generator import ReportGenerator
__all__ = ["EmailReporter", "ReportGenerator"]
+116
View File
@@ -0,0 +1,116 @@
"""
Email Reporter for POTE
Sends formatted reports via SMTP email.
"""
import logging
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import List, Optional
from pote.config import settings
logger = logging.getLogger(__name__)
class EmailReporter:
"""Sends email reports via SMTP."""
def __init__(
self,
smtp_host: Optional[str] = None,
smtp_port: Optional[int] = None,
smtp_user: Optional[str] = None,
smtp_password: Optional[str] = None,
from_email: Optional[str] = None,
):
"""
Initialize email reporter.
If parameters are not provided, will attempt to use settings from config.
"""
self.smtp_host = smtp_host or getattr(settings, "smtp_host", "localhost")
self.smtp_port = smtp_port or getattr(settings, "smtp_port", 587)
self.smtp_user = smtp_user or getattr(settings, "smtp_user", None)
self.smtp_password = smtp_password or getattr(settings, "smtp_password", None)
self.from_email = from_email or getattr(
settings, "from_email", "pote@localhost"
)
def send_report(
self,
to_emails: List[str],
subject: str,
body_text: str,
body_html: Optional[str] = None,
) -> bool:
"""
Send an email report.
Args:
to_emails: List of recipient email addresses
subject: Email subject line
body_text: Plain text email body
body_html: Optional HTML email body
Returns:
True if email sent successfully, False otherwise
"""
try:
msg = MIMEMultipart("alternative")
msg["Subject"] = subject
msg["From"] = self.from_email
msg["To"] = ", ".join(to_emails)
# Attach plain text part
msg.attach(MIMEText(body_text, "plain"))
# Attach HTML part if provided
if body_html:
msg.attach(MIMEText(body_html, "html"))
# Connect to SMTP server and send
with smtplib.SMTP(self.smtp_host, self.smtp_port) as server:
server.ehlo()
if self.smtp_port == 587: # TLS
server.starttls()
server.ehlo()
if self.smtp_user and self.smtp_password:
server.login(self.smtp_user, self.smtp_password)
server.send_message(msg)
logger.info(f"Email sent successfully to {', '.join(to_emails)}")
return True
except Exception as e:
logger.error(f"Failed to send email: {e}")
return False
def test_connection(self) -> bool:
"""
Test SMTP connection.
Returns:
True if connection successful, False otherwise
"""
try:
with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=10) as server:
server.ehlo()
if self.smtp_port == 587:
server.starttls()
server.ehlo()
if self.smtp_user and self.smtp_password:
server.login(self.smtp_user, self.smtp_password)
logger.info("SMTP connection test successful")
return True
except Exception as e:
logger.error(f"SMTP connection test failed: {e}")
return False
+423
View File
@@ -0,0 +1,423 @@
"""
Report Generator for POTE
Generates formatted reports from database data.
"""
import logging
from datetime import date, datetime, timedelta
from typing import Any, Dict, List, Optional
from sqlalchemy import func
from sqlalchemy.orm import Session
from pote.db.models import MarketAlert, Official, Security, Trade
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
from pote.monitoring.pattern_detector import PatternDetector
logger = logging.getLogger(__name__)
class ReportGenerator:
"""Generates various types of reports from database data."""
def __init__(self, session: Session):
self.session = session
self.correlator = DisclosureCorrelator(session)
self.detector = PatternDetector(session)
def generate_daily_summary(
self, report_date: Optional[date] = None
) -> Dict[str, Any]:
"""
Generate a daily summary report.
Args:
report_date: Date to generate report for (defaults to today)
Returns:
Dictionary containing report data
"""
if report_date is None:
report_date = date.today()
start_of_day = datetime.combine(report_date, datetime.min.time())
end_of_day = datetime.combine(report_date, datetime.max.time())
# Count new trades filed today
new_trades = (
self.session.query(Trade).filter(Trade.filing_date == report_date).all()
)
# Count market alerts today
new_alerts = (
self.session.query(MarketAlert)
.filter(
MarketAlert.timestamp >= start_of_day,
MarketAlert.timestamp <= end_of_day,
)
.all()
)
# Get high-severity alerts
critical_alerts = [a for a in new_alerts if a.severity >= 7]
# Get suspicious timing matches
suspicious_trades = []
for trade in new_trades:
analysis = self.correlator.analyze_trade(trade)
if analysis["timing_score"] >= 50:
suspicious_trades.append(analysis)
return {
"date": report_date,
"new_trades_count": len(new_trades),
"new_trades": [
{
"official": t.official.name if t.official else "Unknown",
"ticker": t.security.ticker if t.security else "Unknown",
"side": t.side,
"transaction_date": t.transaction_date,
"value_min": t.value_min,
"value_max": t.value_max,
}
for t in new_trades
],
"market_alerts_count": len(new_alerts),
"critical_alerts_count": len(critical_alerts),
"critical_alerts": [
{
"ticker": a.ticker,
"type": a.alert_type,
"severity": a.severity,
"timestamp": a.timestamp,
"details": a.details,
}
for a in critical_alerts
],
"suspicious_trades_count": len(suspicious_trades),
"suspicious_trades": suspicious_trades,
}
def generate_weekly_summary(self) -> Dict[str, Any]:
"""
Generate a weekly summary report.
Returns:
Dictionary containing report data
"""
week_ago = date.today() - timedelta(days=7)
# Most active officials
active_officials = (
self.session.query(
Official.name, func.count(Trade.id).label("trade_count")
)
.join(Trade)
.filter(Trade.filing_date >= week_ago)
.group_by(Official.id, Official.name)
.order_by(func.count(Trade.id).desc())
.limit(10)
.all()
)
# Most traded securities
active_securities = (
self.session.query(
Security.ticker, func.count(Trade.id).label("trade_count")
)
.join(Trade)
.filter(Trade.filing_date >= week_ago)
.group_by(Security.id, Security.ticker)
.order_by(func.count(Trade.id).desc())
.limit(10)
.all()
)
# Get top suspicious patterns
repeat_offenders = self.detector.identify_repeat_offenders(
days_lookback=7, min_suspicious_trades=2, min_timing_score=40
)
return {
"period_start": week_ago,
"period_end": date.today(),
"most_active_officials": [
{"name": name, "trade_count": count} for name, count in active_officials
],
"most_traded_securities": [
{"ticker": ticker, "trade_count": count}
for ticker, count in active_securities
],
"repeat_offenders_count": len(repeat_offenders),
"repeat_offenders": repeat_offenders[:5], # Top 5
}
def format_as_text(self, report_data: Dict[str, Any], report_type: str) -> str:
"""
Format report data as plain text.
Args:
report_data: Report data dictionary
report_type: Type of report ('daily' or 'weekly')
Returns:
Formatted plain text report
"""
if report_type == "daily":
return self._format_daily_text(report_data)
elif report_type == "weekly":
return self._format_weekly_text(report_data)
else:
return str(report_data)
def _format_daily_text(self, data: Dict[str, Any]) -> str:
"""Format daily report as plain text."""
lines = [
"=" * 70,
f"POTE DAILY REPORT - {data['date']}",
"=" * 70,
"",
"📊 SUMMARY",
f" • New Trades Filed: {data['new_trades_count']}",
f" • Market Alerts: {data['market_alerts_count']}",
f" • Critical Alerts (≥7 severity): {data['critical_alerts_count']}",
f" • Suspicious Timing Trades: {data['suspicious_trades_count']}",
"",
]
if data["new_trades"]:
lines.append("📝 NEW TRADES")
for t in data["new_trades"][:10]: # Limit to 10
lines.append(
f"{t['official']}: {t['side']} {t['ticker']} "
f"(${t['value_min']:,.0f} - ${t['value_max']:,.0f}) "
f"on {t['transaction_date']}"
)
if len(data["new_trades"]) > 10:
lines.append(f" ... and {len(data['new_trades']) - 10} more")
lines.append("")
if data["critical_alerts"]:
lines.append("🚨 CRITICAL MARKET ALERTS")
for a in data["critical_alerts"][:5]:
lines.append(
f"{a['ticker']}: {a['type']} (severity {a['severity']}) "
f"at {a['timestamp'].strftime('%H:%M:%S')}"
)
lines.append("")
if data["suspicious_trades"]:
lines.append("⚠️ SUSPICIOUS TIMING DETECTED")
for st in data["suspicious_trades"][:5]:
lines.append(
f"{st['official_name']}: {st['side']} {st['ticker']} "
f"(Timing Score: {st['timing_score']}/100, "
f"{st['prior_alerts_count']} prior alerts)"
)
lines.append("")
lines.extend(
[
"=" * 70,
"DISCLAIMER: This is for research purposes only. Not investment advice.",
"=" * 70,
]
)
return "\n".join(lines)
def _format_weekly_text(self, data: Dict[str, Any]) -> str:
"""Format weekly report as plain text."""
lines = [
"=" * 70,
f"POTE WEEKLY REPORT - {data['period_start']} to {data['period_end']}",
"=" * 70,
"",
"👥 MOST ACTIVE OFFICIALS",
]
for official in data["most_active_officials"]:
lines.append(f"{official['name']}: {official['trade_count']} trades")
lines.extend(["", "📈 MOST TRADED SECURITIES"])
for security in data["most_traded_securities"]:
lines.append(f"{security['ticker']}: {security['trade_count']} trades")
if data["repeat_offenders"]:
lines.extend(
["", f"⚠️ REPEAT OFFENDERS ({data['repeat_offenders_count']} total)"]
)
for offender in data["repeat_offenders"]:
lines.append(
f"{offender['official_name']}: "
f"{offender['trades_with_timing_advantage']}/{offender['total_trades']} "
f"suspicious trades (avg score: {offender['average_timing_score']:.1f})"
)
lines.extend(
[
"",
"=" * 70,
"DISCLAIMER: This is for research purposes only. Not investment advice.",
"=" * 70,
]
)
return "\n".join(lines)
def format_as_html(self, report_data: Dict[str, Any], report_type: str) -> str:
"""
Format report data as HTML.
Args:
report_data: Report data dictionary
report_type: Type of report ('daily' or 'weekly')
Returns:
Formatted HTML report
"""
if report_type == "daily":
return self._format_daily_html(report_data)
elif report_type == "weekly":
return self._format_weekly_html(report_data)
else:
return f"<pre>{report_data}</pre>"
def _format_daily_html(self, data: Dict[str, Any]) -> str:
"""Format daily report as HTML."""
html = f"""
<html>
<head>
<style>
body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }}
h1 {{ color: #2c3e50; border-bottom: 3px solid #3498db; padding-bottom: 10px; }}
h2 {{ color: #34495e; margin-top: 30px; }}
.summary {{ background: #ecf0f1; padding: 15px; border-radius: 5px; margin: 20px 0; }}
.stat {{ display: inline-block; margin-right: 20px; }}
.alert {{ background: #fff3cd; padding: 10px; margin: 5px 0; border-left: 4px solid #ffc107; }}
.critical {{ background: #f8d7da; border-left: 4px solid #dc3545; }}
.trade {{ background: #d1ecf1; padding: 10px; margin: 5px 0; border-left: 4px solid #17a2b8; }}
.disclaimer {{ background: #e9ecef; padding: 10px; margin-top: 30px; font-size: 0.9em; border-left: 4px solid #6c757d; }}
</style>
</head>
<body>
<h1>POTE Daily Report - {data['date']}</h1>
<div class="summary">
<h2>📊 Summary</h2>
<div class="stat"><strong>New Trades:</strong> {data['new_trades_count']}</div>
<div class="stat"><strong>Market Alerts:</strong> {data['market_alerts_count']}</div>
<div class="stat"><strong>Critical Alerts:</strong> {data['critical_alerts_count']}</div>
<div class="stat"><strong>Suspicious Trades:</strong> {data['suspicious_trades_count']}</div>
</div>
"""
if data["new_trades"]:
html += "<h2>📝 New Trades</h2>"
for t in data["new_trades"][:10]:
html += f"""
<div class="trade">
<strong>{t['official']}</strong>: {t['side']} {t['ticker']}
(${t['value_min']:,.0f} - ${t['value_max']:,.0f}) on {t['transaction_date']}
</div>
"""
if data["critical_alerts"]:
html += "<h2>🚨 Critical Market Alerts</h2>"
for a in data["critical_alerts"][:5]:
html += f"""
<div class="alert critical">
<strong>{a['ticker']}</strong>: {a['type']} (severity {a['severity']})
at {a['timestamp'].strftime('%H:%M:%S')}
</div>
"""
if data["suspicious_trades"]:
html += "<h2>⚠️ Suspicious Timing Detected</h2>"
for st in data["suspicious_trades"][:5]:
html += f"""
<div class="alert">
<strong>{st['official_name']}</strong>: {st['side']} {st['ticker']}<br>
Timing Score: {st['timing_score']}/100 ({st['prior_alerts_count']} prior alerts)
</div>
"""
html += """
<div class="disclaimer">
<strong>DISCLAIMER:</strong> This is for research purposes only. Not investment advice.
</div>
</body>
</html>
"""
return html
def _format_weekly_html(self, data: Dict[str, Any]) -> str:
"""Format weekly report as HTML."""
html = f"""
<html>
<head>
<style>
body {{ font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }}
h1 {{ color: #2c3e50; border-bottom: 3px solid #3498db; padding-bottom: 10px; }}
h2 {{ color: #34495e; margin-top: 30px; }}
table {{ width: 100%; border-collapse: collapse; margin: 20px 0; }}
th, td {{ padding: 10px; text-align: left; border-bottom: 1px solid #ddd; }}
th {{ background: #3498db; color: white; }}
.disclaimer {{ background: #e9ecef; padding: 10px; margin-top: 30px; font-size: 0.9em; border-left: 4px solid #6c757d; }}
</style>
</head>
<body>
<h1>POTE Weekly Report</h1>
<p><strong>Period:</strong> {data['period_start']} to {data['period_end']}</p>
<h2>👥 Most Active Officials</h2>
<table>
<tr><th>Official</th><th>Trade Count</th></tr>
"""
for official in data["most_active_officials"]:
html += f"<tr><td>{official['name']}</td><td>{official['trade_count']}</td></tr>"
html += """
</table>
<h2>📈 Most Traded Securities</h2>
<table>
<tr><th>Ticker</th><th>Trade Count</th></tr>
"""
for security in data["most_traded_securities"]:
html += f"<tr><td>{security['ticker']}</td><td>{security['trade_count']}</td></tr>"
html += "</table>"
if data["repeat_offenders"]:
html += f"""
<h2>⚠️ Repeat Offenders ({data['repeat_offenders_count']} total)</h2>
<table>
<tr><th>Official</th><th>Suspicious Trades</th><th>Total Trades</th><th>Avg Score</th></tr>
"""
for offender in data["repeat_offenders"]:
html += f"""
<tr>
<td>{offender['official_name']}</td>
<td>{offender['trades_with_timing_advantage']}</td>
<td>{offender['total_trades']}</td>
<td>{offender['average_timing_score']:.1f}</td>
</tr>
"""
html += "</table>"
html += """
<div class="disclaimer">
<strong>DISCLAIMER:</strong> This is for research purposes only. Not investment advice.
</div>
</body>
</html>
"""
return html