POTE/scripts/analyze_disclosure_timing.py
ilia 0d8d85adc1 Add complete automation, reporting, and CI/CD system
Features Added:
==============

📧 EMAIL REPORTING SYSTEM:
- EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom)
- ReportGenerator: Generate daily/weekly summaries with HTML/text formatting
- Configurable via .env (SMTP_HOST, SMTP_PORT, etc.)
- Scripts: send_daily_report.py, send_weekly_report.py

🤖 AUTOMATED RUNS:
- automated_daily_run.sh: Full daily ETL pipeline + reporting
- automated_weekly_run.sh: Weekly pattern analysis + reports
- setup_cron.sh: Interactive cron job setup (5-minute setup)
- Logs saved to ~/logs/ with automatic cleanup

🔍 HEALTH CHECKS:
- health_check.py: System health monitoring
- Checks: DB connection, data freshness, counts, recent alerts
- JSON output for programmatic use
- Exit codes for monitoring integration

🚀 CI/CD PIPELINE:
- .github/workflows/ci.yml: Full CI/CD pipeline
- GitHub Actions / Gitea Actions compatible
- Jobs: lint & test, security scan, dependency scan, Docker build
- PostgreSQL service for integration tests
- 93 tests passing in CI

📚 COMPREHENSIVE DOCUMENTATION:
- AUTOMATION_QUICKSTART.md: 5-minute email setup guide
- docs/12_automation_and_reporting.md: Full automation guide
- Updated README.md with automation links
- Deployment → Production workflow guide

🛠️ IMPROVEMENTS:
- All shell scripts made executable
- Environment variable examples in .env.example
- Report logs saved with timestamps
- 30-day log retention with auto-cleanup
- Health checks can be scheduled via cron

WHAT THIS ENABLES:
==================
After deployment, users can:
1. Set up automated daily/weekly email reports (5 min)
2. Receive HTML+text emails with:
   - New trades, market alerts, suspicious timing
   - Weekly patterns, rankings, repeat offenders
3. Monitor system health automatically
4. Run full CI/CD pipeline on every commit
5. Deploy with confidence (tests + security scans)

USAGE:
======
# One-time setup (on deployed server)
./scripts/setup_cron.sh

# Or manually send reports
python scripts/send_daily_report.py --to user@example.com
python scripts/send_weekly_report.py --to user@example.com

# Check system health
python scripts/health_check.py

See AUTOMATION_QUICKSTART.md for full instructions.

93 tests passing | Full CI/CD | Email reports ready
2025-12-15 15:34:31 -05:00

221 lines
7.8 KiB
Python
Executable File

#!/usr/bin/env python
"""
Analyze congressional trade timing vs market alerts.
Identifies suspicious timing patterns and potential insider trading.
"""
import click
from pathlib import Path
from tabulate import tabulate
from pote.db import get_session
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
@click.command()
@click.option("--days", default=30, help="Analyze trades filed in last N days")
@click.option("--min-score", default=50, help="Minimum timing score to report (0-100)")
@click.option("--official", help="Analyze specific official by name")
@click.option("--ticker", help="Analyze specific ticker")
@click.option("--output", help="Save report to file")
@click.option("--format", type=click.Choice(["text", "json"]), default="text")
def main(days, min_score, official, ticker, output, format):
"""Analyze disclosure timing and detect suspicious patterns."""
session = next(get_session())
correlator = DisclosureCorrelator(session)
if official:
# Analyze specific official
from pote.db.models import Official
official_obj = session.query(Official).filter(
Official.name.ilike(f"%{official}%")
).first()
if not official_obj:
click.echo(f"❌ Official '{official}' not found")
return
click.echo(f"\n📊 Analyzing {official_obj.name}...\n")
result = correlator.get_official_timing_pattern(official_obj.id)
report = format_official_report(result)
click.echo(report)
elif ticker:
# Analyze specific ticker
click.echo(f"\n📊 Analyzing trades in {ticker.upper()}...\n")
result = correlator.get_ticker_timing_analysis(ticker.upper())
report = format_ticker_report(result)
click.echo(report)
else:
# Analyze recent disclosures
click.echo(f"\n🔍 Analyzing trades filed in last {days} days...")
click.echo(f" Minimum timing score: {min_score}\n")
suspicious_trades = correlator.analyze_recent_disclosures(
days=days,
min_timing_score=min_score
)
if not suspicious_trades:
click.echo(f"✅ No trades found with timing score >= {min_score}")
return
report = format_suspicious_trades_report(suspicious_trades)
click.echo(report)
# Save to file if requested
if output:
Path(output).write_text(report)
click.echo(f"\n💾 Report saved to {output}")
def format_suspicious_trades_report(trades):
"""Format suspicious trades as text report."""
lines = [
"=" * 100,
f" SUSPICIOUS TRADING TIMING ANALYSIS",
f" {len(trades)} Trades with Timing Advantages Detected",
"=" * 100,
"",
]
for i, trade in enumerate(trades, 1):
# Determine alert level
if trade.get("highly_suspicious"):
alert_emoji = "🚨"
level = "HIGHLY SUSPICIOUS"
elif trade["suspicious"]:
alert_emoji = "🔴"
level = "SUSPICIOUS"
else:
alert_emoji = "🟡"
level = "NOTABLE"
lines.extend([
"" * 100,
f"{alert_emoji} #{i} - {level} (Timing Score: {trade['timing_score']}/100)",
"" * 100,
f"Official: {trade['official_name']}",
f"Ticker: {trade['ticker']}",
f"Side: {trade['side'].upper()}",
f"Trade Date: {trade['transaction_date']}",
f"Filed Date: {trade['filing_date']}",
f"Value: {trade['value_range']}",
"",
f"📊 Timing Analysis:",
f" Prior Alerts: {trade['alert_count']}",
f" Recent Alerts (7d): {trade['recent_alert_count']}",
f" High Severity: {trade['high_severity_count']}",
f" Avg Severity: {trade['avg_severity']}/10",
"",
f"💡 Assessment: {trade['reason']}",
"",
])
if trade['prior_alerts']:
lines.append("🔔 Prior Market Alerts:")
alert_table = []
for alert in trade['prior_alerts'][:5]: # Top 5
alert_table.append([
alert['timestamp'],
alert['alert_type'].replace('_', ' ').title(),
f"{alert['severity']}/10",
f"{alert['days_before_trade']} days before",
])
lines.append(tabulate(
alert_table,
headers=["Timestamp", "Type", "Severity", "Timing"],
tablefmt="simple"
))
lines.append("")
lines.extend([
"=" * 100,
"📈 SUMMARY",
"=" * 100,
f"Total Suspicious Trades: {len(trades)}",
f"Highly Suspicious: {sum(1 for t in trades if t.get('highly_suspicious'))}",
f"Average Timing Score: {sum(t['timing_score'] for t in trades) / len(trades):.2f}/100",
"",
"⚠️ IMPORTANT:",
" This analysis is for research and transparency purposes only.",
" High timing scores suggest potential issues but are not definitive proof.",
" Further investigation may be warranted for highly suspicious patterns.",
"",
"=" * 100,
])
return "\n".join(lines)
def format_official_report(result):
"""Format official timing pattern report."""
lines = [
"=" * 80,
f" OFFICIAL TIMING PATTERN ANALYSIS",
"=" * 80,
"",
f"Trade Count: {result['trade_count']}",
f"With Prior Alerts: {result['trades_with_prior_alerts']} ({result['trades_with_prior_alerts']/result['trade_count']*100:.1f}%)" if result['trade_count'] > 0 else "",
f"Suspicious Trades: {result['suspicious_trade_count']}",
f"Highly Suspicious: {result['highly_suspicious_count']}",
f"Average Timing Score: {result['avg_timing_score']}/100",
"",
f"📊 Pattern: {result['pattern']}",
"",
]
if result.get('analyses'):
# Show top suspicious trades
suspicious = [a for a in result['analyses'] if a['suspicious']]
if suspicious:
lines.append("🚨 Most Suspicious Trades:")
for trade in suspicious[:5]:
lines.append(
f" {trade['ticker']:6s} {trade['side']:4s} on {trade['transaction_date']} "
f"(Score: {trade['timing_score']:.0f}/100, {trade['alert_count']} alerts)"
)
lines.append("")
lines.append("=" * 80)
return "\n".join(lines)
def format_ticker_report(result):
"""Format ticker timing analysis report."""
lines = [
"=" * 80,
f" TICKER TIMING ANALYSIS: {result['ticker']}",
"=" * 80,
"",
f"Total Trades: {result['trade_count']}",
f"With Prior Alerts: {result['trades_with_alerts']}",
f"Suspicious Count: {result['suspicious_count']}",
f"Average Timing Score: {result['avg_timing_score']}/100",
"",
]
if result.get('analyses'):
lines.append("📊 Recent Trades:")
for trade in result['analyses'][:10]:
emoji = "🚨" if trade.get('highly_suspicious') else "🔴" if trade['suspicious'] else "🟡" if trade['alert_count'] > 0 else ""
lines.append(
f" {emoji} {trade['official_name']:25s} {trade['side']:4s} on {trade['transaction_date']} "
f"(Score: {trade['timing_score']:.0f}/100)"
)
lines.append("")
lines.append("=" * 80)
return "\n".join(lines)
if __name__ == "__main__":
main()