Features Added: ============== 📧 EMAIL REPORTING SYSTEM: - EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom) - ReportGenerator: Generate daily/weekly summaries with HTML/text formatting - Configurable via .env (SMTP_HOST, SMTP_PORT, etc.) - Scripts: send_daily_report.py, send_weekly_report.py 🤖 AUTOMATED RUNS: - automated_daily_run.sh: Full daily ETL pipeline + reporting - automated_weekly_run.sh: Weekly pattern analysis + reports - setup_cron.sh: Interactive cron job setup (5-minute setup) - Logs saved to ~/logs/ with automatic cleanup 🔍 HEALTH CHECKS: - health_check.py: System health monitoring - Checks: DB connection, data freshness, counts, recent alerts - JSON output for programmatic use - Exit codes for monitoring integration 🚀 CI/CD PIPELINE: - .github/workflows/ci.yml: Full CI/CD pipeline - GitHub Actions / Gitea Actions compatible - Jobs: lint & test, security scan, dependency scan, Docker build - PostgreSQL service for integration tests - 93 tests passing in CI 📚 COMPREHENSIVE DOCUMENTATION: - AUTOMATION_QUICKSTART.md: 5-minute email setup guide - docs/12_automation_and_reporting.md: Full automation guide - Updated README.md with automation links - Deployment → Production workflow guide 🛠️ IMPROVEMENTS: - All shell scripts made executable - Environment variable examples in .env.example - Report logs saved with timestamps - 30-day log retention with auto-cleanup - Health checks can be scheduled via cron WHAT THIS ENABLES: ================== After deployment, users can: 1. Set up automated daily/weekly email reports (5 min) 2. Receive HTML+text emails with: - New trades, market alerts, suspicious timing - Weekly patterns, rankings, repeat offenders 3. Monitor system health automatically 4. Run full CI/CD pipeline on every commit 5. Deploy with confidence (tests + security scans) USAGE: ====== # One-time setup (on deployed server) ./scripts/setup_cron.sh # Or manually send reports python scripts/send_daily_report.py --to user@example.com python scripts/send_weekly_report.py --to user@example.com # Check system health python scripts/health_check.py See AUTOMATION_QUICKSTART.md for full instructions. 93 tests passing | Full CI/CD | Email reports ready
221 lines
7.8 KiB
Python
Executable File
221 lines
7.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""
|
|
Analyze congressional trade timing vs market alerts.
|
|
Identifies suspicious timing patterns and potential insider trading.
|
|
"""
|
|
|
|
import click
|
|
from pathlib import Path
|
|
from tabulate import tabulate
|
|
|
|
from pote.db import get_session
|
|
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
|
|
|
|
|
|
@click.command()
|
|
@click.option("--days", default=30, help="Analyze trades filed in last N days")
|
|
@click.option("--min-score", default=50, help="Minimum timing score to report (0-100)")
|
|
@click.option("--official", help="Analyze specific official by name")
|
|
@click.option("--ticker", help="Analyze specific ticker")
|
|
@click.option("--output", help="Save report to file")
|
|
@click.option("--format", type=click.Choice(["text", "json"]), default="text")
|
|
def main(days, min_score, official, ticker, output, format):
|
|
"""Analyze disclosure timing and detect suspicious patterns."""
|
|
|
|
session = next(get_session())
|
|
correlator = DisclosureCorrelator(session)
|
|
|
|
if official:
|
|
# Analyze specific official
|
|
from pote.db.models import Official
|
|
official_obj = session.query(Official).filter(
|
|
Official.name.ilike(f"%{official}%")
|
|
).first()
|
|
|
|
if not official_obj:
|
|
click.echo(f"❌ Official '{official}' not found")
|
|
return
|
|
|
|
click.echo(f"\n📊 Analyzing {official_obj.name}...\n")
|
|
result = correlator.get_official_timing_pattern(official_obj.id)
|
|
|
|
report = format_official_report(result)
|
|
click.echo(report)
|
|
|
|
elif ticker:
|
|
# Analyze specific ticker
|
|
click.echo(f"\n📊 Analyzing trades in {ticker.upper()}...\n")
|
|
result = correlator.get_ticker_timing_analysis(ticker.upper())
|
|
|
|
report = format_ticker_report(result)
|
|
click.echo(report)
|
|
|
|
else:
|
|
# Analyze recent disclosures
|
|
click.echo(f"\n🔍 Analyzing trades filed in last {days} days...")
|
|
click.echo(f" Minimum timing score: {min_score}\n")
|
|
|
|
suspicious_trades = correlator.analyze_recent_disclosures(
|
|
days=days,
|
|
min_timing_score=min_score
|
|
)
|
|
|
|
if not suspicious_trades:
|
|
click.echo(f"✅ No trades found with timing score >= {min_score}")
|
|
return
|
|
|
|
report = format_suspicious_trades_report(suspicious_trades)
|
|
click.echo(report)
|
|
|
|
# Save to file if requested
|
|
if output:
|
|
Path(output).write_text(report)
|
|
click.echo(f"\n💾 Report saved to {output}")
|
|
|
|
|
|
def format_suspicious_trades_report(trades):
|
|
"""Format suspicious trades as text report."""
|
|
lines = [
|
|
"=" * 100,
|
|
f" SUSPICIOUS TRADING TIMING ANALYSIS",
|
|
f" {len(trades)} Trades with Timing Advantages Detected",
|
|
"=" * 100,
|
|
"",
|
|
]
|
|
|
|
for i, trade in enumerate(trades, 1):
|
|
# Determine alert level
|
|
if trade.get("highly_suspicious"):
|
|
alert_emoji = "🚨"
|
|
level = "HIGHLY SUSPICIOUS"
|
|
elif trade["suspicious"]:
|
|
alert_emoji = "🔴"
|
|
level = "SUSPICIOUS"
|
|
else:
|
|
alert_emoji = "🟡"
|
|
level = "NOTABLE"
|
|
|
|
lines.extend([
|
|
"─" * 100,
|
|
f"{alert_emoji} #{i} - {level} (Timing Score: {trade['timing_score']}/100)",
|
|
"─" * 100,
|
|
f"Official: {trade['official_name']}",
|
|
f"Ticker: {trade['ticker']}",
|
|
f"Side: {trade['side'].upper()}",
|
|
f"Trade Date: {trade['transaction_date']}",
|
|
f"Filed Date: {trade['filing_date']}",
|
|
f"Value: {trade['value_range']}",
|
|
"",
|
|
f"📊 Timing Analysis:",
|
|
f" Prior Alerts: {trade['alert_count']}",
|
|
f" Recent Alerts (7d): {trade['recent_alert_count']}",
|
|
f" High Severity: {trade['high_severity_count']}",
|
|
f" Avg Severity: {trade['avg_severity']}/10",
|
|
"",
|
|
f"💡 Assessment: {trade['reason']}",
|
|
"",
|
|
])
|
|
|
|
if trade['prior_alerts']:
|
|
lines.append("🔔 Prior Market Alerts:")
|
|
alert_table = []
|
|
for alert in trade['prior_alerts'][:5]: # Top 5
|
|
alert_table.append([
|
|
alert['timestamp'],
|
|
alert['alert_type'].replace('_', ' ').title(),
|
|
f"{alert['severity']}/10",
|
|
f"{alert['days_before_trade']} days before",
|
|
])
|
|
|
|
lines.append(tabulate(
|
|
alert_table,
|
|
headers=["Timestamp", "Type", "Severity", "Timing"],
|
|
tablefmt="simple"
|
|
))
|
|
lines.append("")
|
|
|
|
lines.extend([
|
|
"=" * 100,
|
|
"📈 SUMMARY",
|
|
"=" * 100,
|
|
f"Total Suspicious Trades: {len(trades)}",
|
|
f"Highly Suspicious: {sum(1 for t in trades if t.get('highly_suspicious'))}",
|
|
f"Average Timing Score: {sum(t['timing_score'] for t in trades) / len(trades):.2f}/100",
|
|
"",
|
|
"⚠️ IMPORTANT:",
|
|
" This analysis is for research and transparency purposes only.",
|
|
" High timing scores suggest potential issues but are not definitive proof.",
|
|
" Further investigation may be warranted for highly suspicious patterns.",
|
|
"",
|
|
"=" * 100,
|
|
])
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def format_official_report(result):
|
|
"""Format official timing pattern report."""
|
|
lines = [
|
|
"=" * 80,
|
|
f" OFFICIAL TIMING PATTERN ANALYSIS",
|
|
"=" * 80,
|
|
"",
|
|
f"Trade Count: {result['trade_count']}",
|
|
f"With Prior Alerts: {result['trades_with_prior_alerts']} ({result['trades_with_prior_alerts']/result['trade_count']*100:.1f}%)" if result['trade_count'] > 0 else "",
|
|
f"Suspicious Trades: {result['suspicious_trade_count']}",
|
|
f"Highly Suspicious: {result['highly_suspicious_count']}",
|
|
f"Average Timing Score: {result['avg_timing_score']}/100",
|
|
"",
|
|
f"📊 Pattern: {result['pattern']}",
|
|
"",
|
|
]
|
|
|
|
if result.get('analyses'):
|
|
# Show top suspicious trades
|
|
suspicious = [a for a in result['analyses'] if a['suspicious']]
|
|
if suspicious:
|
|
lines.append("🚨 Most Suspicious Trades:")
|
|
for trade in suspicious[:5]:
|
|
lines.append(
|
|
f" {trade['ticker']:6s} {trade['side']:4s} on {trade['transaction_date']} "
|
|
f"(Score: {trade['timing_score']:.0f}/100, {trade['alert_count']} alerts)"
|
|
)
|
|
lines.append("")
|
|
|
|
lines.append("=" * 80)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def format_ticker_report(result):
|
|
"""Format ticker timing analysis report."""
|
|
lines = [
|
|
"=" * 80,
|
|
f" TICKER TIMING ANALYSIS: {result['ticker']}",
|
|
"=" * 80,
|
|
"",
|
|
f"Total Trades: {result['trade_count']}",
|
|
f"With Prior Alerts: {result['trades_with_alerts']}",
|
|
f"Suspicious Count: {result['suspicious_count']}",
|
|
f"Average Timing Score: {result['avg_timing_score']}/100",
|
|
"",
|
|
]
|
|
|
|
if result.get('analyses'):
|
|
lines.append("📊 Recent Trades:")
|
|
for trade in result['analyses'][:10]:
|
|
emoji = "🚨" if trade.get('highly_suspicious') else "🔴" if trade['suspicious'] else "🟡" if trade['alert_count'] > 0 else "✅"
|
|
lines.append(
|
|
f" {emoji} {trade['official_name']:25s} {trade['side']:4s} on {trade['transaction_date']} "
|
|
f"(Score: {trade['timing_score']:.0f}/100)"
|
|
)
|
|
lines.append("")
|
|
|
|
lines.append("=" * 80)
|
|
return "\n".join(lines)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
|