POTE/scripts/generate_trading_report.py
ilia 0d8d85adc1 Add complete automation, reporting, and CI/CD system
Features Added:
==============

📧 EMAIL REPORTING SYSTEM:
- EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom)
- ReportGenerator: Generate daily/weekly summaries with HTML/text formatting
- Configurable via .env (SMTP_HOST, SMTP_PORT, etc.)
- Scripts: send_daily_report.py, send_weekly_report.py

🤖 AUTOMATED RUNS:
- automated_daily_run.sh: Full daily ETL pipeline + reporting
- automated_weekly_run.sh: Weekly pattern analysis + reports
- setup_cron.sh: Interactive cron job setup (5-minute setup)
- Logs saved to ~/logs/ with automatic cleanup

🔍 HEALTH CHECKS:
- health_check.py: System health monitoring
- Checks: DB connection, data freshness, counts, recent alerts
- JSON output for programmatic use
- Exit codes for monitoring integration

🚀 CI/CD PIPELINE:
- .github/workflows/ci.yml: Full CI/CD pipeline
- GitHub Actions / Gitea Actions compatible
- Jobs: lint & test, security scan, dependency scan, Docker build
- PostgreSQL service for integration tests
- 93 tests passing in CI

📚 COMPREHENSIVE DOCUMENTATION:
- AUTOMATION_QUICKSTART.md: 5-minute email setup guide
- docs/12_automation_and_reporting.md: Full automation guide
- Updated README.md with automation links
- Deployment → Production workflow guide

🛠️ IMPROVEMENTS:
- All shell scripts made executable
- Environment variable examples in .env.example
- Report logs saved with timestamps
- 30-day log retention with auto-cleanup
- Health checks can be scheduled via cron

WHAT THIS ENABLES:
==================
After deployment, users can:
1. Set up automated daily/weekly email reports (5 min)
2. Receive HTML+text emails with:
   - New trades, market alerts, suspicious timing
   - Weekly patterns, rankings, repeat offenders
3. Monitor system health automatically
4. Run full CI/CD pipeline on every commit
5. Deploy with confidence (tests + security scans)

USAGE:
======
# One-time setup (on deployed server)
./scripts/setup_cron.sh

# Or manually send reports
python scripts/send_daily_report.py --to user@example.com
python scripts/send_weekly_report.py --to user@example.com

# Check system health
python scripts/health_check.py

See AUTOMATION_QUICKSTART.md for full instructions.

93 tests passing | Full CI/CD | Email reports ready
2025-12-15 15:34:31 -05:00

308 lines
9.5 KiB
Python
Executable File

#!/usr/bin/env python
"""
Generate trading report for watched Congress members.
Shows NEW trades filed recently.
"""
import json
from datetime import date, timedelta
from pathlib import Path
from decimal import Decimal
import click
from sqlalchemy import text
from tabulate import tabulate
from pote.db import get_session
from pote.db.models import Official, Security, Trade
def load_watchlist():
"""Load watchlist of officials to monitor."""
config_path = Path(__file__).parent.parent / "config" / "watchlist.json"
if not config_path.exists():
print("⚠️ No watchlist found. Creating default...")
import subprocess
subprocess.run([
"python",
str(Path(__file__).parent / "fetch_congress_members.py"),
"--create"
])
with open(config_path) as f:
return json.load(f)
def get_new_trades(session, days=7, watchlist=None):
"""
Get trades filed in the last N days.
Args:
session: Database session
days: Look back this many days
watchlist: List of official names to filter (None = all)
"""
since_date = date.today() - timedelta(days=days)
query = text("""
SELECT
o.name,
o.chamber,
o.party,
o.state,
s.ticker,
s.name as company,
s.sector,
t.side,
t.transaction_date,
t.filing_date,
t.value_min,
t.value_max,
t.created_at
FROM trades t
JOIN officials o ON t.official_id = o.id
JOIN securities s ON t.security_id = s.id
WHERE t.created_at >= :since_date
ORDER BY t.created_at DESC, t.transaction_date DESC
""")
result = session.execute(query, {"since_date": since_date})
trades = result.fetchall()
# Filter by watchlist if provided
if watchlist:
watchlist_names = {m['name'].lower() for m in watchlist}
trades = [t for t in trades if t[0].lower() in watchlist_names]
return trades
def format_value(vmin, vmax):
"""Format trade value range."""
if vmax and vmax > vmin:
return f"${float(vmin):,.0f} - ${float(vmax):,.0f}"
else:
return f"${float(vmin):,.0f}+"
def generate_report(trades, format="text"):
"""Generate formatted report."""
if not trades:
return "📭 No new trades found."
if format == "text":
return generate_text_report(trades)
elif format == "html":
return generate_html_report(trades)
elif format == "json":
return generate_json_report(trades)
else:
return generate_text_report(trades)
def generate_text_report(trades):
"""Generate text report."""
report = []
report.append(f"\n{'='*80}")
report.append(f" CONGRESSIONAL TRADING REPORT")
report.append(f" {len(trades)} New Trades")
report.append(f" Generated: {date.today()}")
report.append(f"{'='*80}\n")
# Group by official
by_official = {}
for trade in trades:
name = trade[0]
if name not in by_official:
by_official[name] = []
by_official[name].append(trade)
# Generate section for each official
for official_name, official_trades in by_official.items():
# Header
first_trade = official_trades[0]
chamber, party, state = first_trade[1], first_trade[2], first_trade[3]
report.append(f"\n{''*80}")
report.append(f"👤 {official_name} ({party[0]}-{state}, {chamber})")
report.append(f"{''*80}")
# Trades table
table_data = []
for t in official_trades:
ticker, company, sector = t[4], t[5], t[6]
side, txn_date, filing_date = t[7], t[8], t[9]
vmin, vmax = t[10], t[11]
# Color code side
side_emoji = "🟢 BUY" if side.lower() == "buy" else "🔴 SELL"
table_data.append([
side_emoji,
ticker,
f"{company[:30]}..." if company and len(company) > 30 else (company or ""),
sector or "",
format_value(vmin, vmax),
str(txn_date),
str(filing_date),
])
table = tabulate(
table_data,
headers=["Side", "Ticker", "Company", "Sector", "Value", "Trade Date", "Filed"],
tablefmt="simple"
)
report.append(table)
report.append("")
# Summary statistics
report.append(f"\n{'='*80}")
report.append("📊 SUMMARY")
report.append(f"{'='*80}")
total_buys = sum(1 for t in trades if t[7].lower() == "buy")
total_sells = sum(1 for t in trades if t[7].lower() == "sell")
unique_tickers = len(set(t[4] for t in trades))
unique_officials = len(by_official)
# Top tickers
ticker_counts = {}
for t in trades:
ticker = t[4]
ticker_counts[ticker] = ticker_counts.get(ticker, 0) + 1
top_tickers = sorted(ticker_counts.items(), key=lambda x: x[1], reverse=True)[:5]
report.append(f"\nTotal Trades: {len(trades)}")
report.append(f" Buys: {total_buys}")
report.append(f" Sells: {total_sells}")
report.append(f"Unique Officials: {unique_officials}")
report.append(f"Unique Tickers: {unique_tickers}")
report.append(f"\nTop Tickers:")
for ticker, count in top_tickers:
report.append(f" {ticker:6s} - {count} trades")
report.append(f"\n{'='*80}\n")
return "\n".join(report)
def generate_html_report(trades):
"""Generate HTML email report."""
html = [
"<html><head><style>",
"body { font-family: Arial, sans-serif; }",
"table { border-collapse: collapse; width: 100%; margin: 20px 0; }",
"th { background: #333; color: white; padding: 10px; text-align: left; }",
"td { border: 1px solid #ddd; padding: 8px; }",
".buy { color: green; font-weight: bold; }",
".sell { color: red; font-weight: bold; }",
"</style></head><body>",
f"<h1>Congressional Trading Report</h1>",
f"<p><strong>{len(trades)} New Trades</strong> | Generated: {date.today()}</p>",
]
# Group by official
by_official = {}
for trade in trades:
name = trade[0]
if name not in by_official:
by_official[name] = []
by_official[name].append(trade)
for official_name, official_trades in by_official.items():
first_trade = official_trades[0]
chamber, party, state = first_trade[1], first_trade[2], first_trade[3]
html.append(f"<h2>{official_name} ({party[0]}-{state}, {chamber})</h2>")
html.append("<table>")
html.append("<tr><th>Side</th><th>Ticker</th><th>Company</th><th>Value</th><th>Trade Date</th><th>Filed</th></tr>")
for t in official_trades:
ticker, company, sector = t[4], t[5], t[6]
side, txn_date, filing_date = t[7], t[8], t[9]
vmin, vmax = t[10], t[11]
side_class = "buy" if side.lower() == "buy" else "sell"
side_text = "BUY" if side.lower() == "buy" else "SELL"
html.append(f"<tr>")
html.append(f"<td class='{side_class}'>{side_text}</td>")
html.append(f"<td><strong>{ticker}</strong></td>")
html.append(f"<td>{company or ''}</td>")
html.append(f"<td>{format_value(vmin, vmax)}</td>")
html.append(f"<td>{txn_date}</td>")
html.append(f"<td>{filing_date}</td>")
html.append(f"</tr>")
html.append("</table>")
html.append("</body></html>")
return "\n".join(html)
def generate_json_report(trades):
"""Generate JSON report for programmatic use."""
import json
trades_list = []
for t in trades:
trades_list.append({
"official": t[0],
"chamber": t[1],
"party": t[2],
"state": t[3],
"ticker": t[4],
"company": t[5],
"sector": t[6],
"side": t[7],
"transaction_date": str(t[8]),
"filing_date": str(t[9]),
"value_min": float(t[10]),
"value_max": float(t[11]) if t[11] else None,
})
return json.dumps({
"generated": str(date.today()),
"trade_count": len(trades),
"trades": trades_list
}, indent=2)
@click.command()
@click.option("--days", default=7, help="Look back this many days")
@click.option("--watchlist-only", is_flag=True, help="Only show trades from watchlist")
@click.option("--format", type=click.Choice(["text", "html", "json"]), default="text")
@click.option("--output", help="Output file (default: stdout)")
def main(days, watchlist_only, format, output):
"""Generate trading report for Congress members."""
session = next(get_session())
# Load watchlist if requested
watchlist = None
if watchlist_only:
watchlist = load_watchlist()
print(f"📋 Filtering for {len(watchlist)} officials on watchlist\n")
# Get trades
print(f"🔍 Fetching trades from last {days} days...")
trades = get_new_trades(session, days=days, watchlist=watchlist)
# Generate report
report = generate_report(trades, format=format)
# Output
if output:
Path(output).write_text(report)
print(f"✅ Report saved to {output}")
else:
print(report)
if __name__ == "__main__":
main()