Add watchlist system and pre-market trading reports

New Features:
- Watchlist system for tracking specific Congress members
- Trading report generation with multiple formats
- Pre-market-close automated updates (3 PM)

New Scripts:
- scripts/fetch_congress_members.py: Manage watchlist
  * 29 known active traders (curated list)
  * Optional ProPublica API integration (all 535 members)
  * Create/view/manage watchlist

- scripts/generate_trading_report.py: Generate trading reports
  * Filter by watchlist or show all
  * Multiple formats: text, HTML, JSON
  * Summary statistics (buys/sells, top tickers)
  * Color-coded output (🟢 BUY, 🔴 SELL)

- scripts/pre_market_close_update.sh: 3 PM automation
  * Quick fetch of latest trades
  * Enrichment of new securities
  * Generate and display report
  * Saves to reports/ directory

Documentation:
- WATCHLIST_GUIDE.md: Complete guide
  * List of 29 known active traders
  * How to create/customize watchlist
  * Schedule options (pre-market, post-market)
  * Email setup (optional)
  * FAQ and examples

Known Active Traders Include:
Senate: Tuberville, Rand Paul, Mark Warner, Rick Scott
House: Pelosi, Crenshaw, MTG, Gottheimer, Brian Higgins

Use Cases:
 Daily reports at 3 PM (1 hour before close)
 See what Congress bought/sold recently
 Track specific members you care about
 Export to HTML/JSON for further analysis
This commit is contained in:
ilia
2025-12-15 15:00:42 -05:00
parent 3a89c1e6d2
commit 8ba9d7ffdd
4 changed files with 964 additions and 0 deletions
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env python
"""
Fetch current members of Congress.
Creates a watchlist of active members.
"""
import json
import requests
from pathlib import Path
# Congress.gov API (no key required for basic info)
# or ProPublica Congress API (requires free key)
def fetch_from_propublica():
"""
Fetch from ProPublica Congress API.
Free API key: https://www.propublica.org/datastore/api/propublica-congress-api
"""
API_KEY = "YOUR_API_KEY" # Get free key from ProPublica
headers = {"X-API-Key": API_KEY}
# Get current Congress (118th = 2023-2025, 119th = 2025-2027)
members = []
# Senate
senate_url = "https://api.propublica.org/congress/v1/118/senate/members.json"
response = requests.get(senate_url, headers=headers)
if response.ok:
data = response.json()
for member in data['results'][0]['members']:
members.append({
'name': f"{member['first_name']} {member['last_name']}",
'chamber': 'Senate',
'party': member['party'],
'state': member['state'],
})
# House
house_url = "https://api.propublica.org/congress/v1/118/house/members.json"
response = requests.get(house_url, headers=headers)
if response.ok:
data = response.json()
for member in data['results'][0]['members']:
members.append({
'name': f"{member['first_name']} {member['last_name']}",
'chamber': 'House',
'party': member['party'],
'state': member['state'],
})
return members
def get_known_active_traders():
"""
Manually curated list of Congress members known for active trading.
Based on public reporting and analysis.
"""
return [
# === SENATE ===
{"name": "Tommy Tuberville", "chamber": "Senate", "party": "Republican", "state": "AL"},
{"name": "Rand Paul", "chamber": "Senate", "party": "Republican", "state": "KY"},
{"name": "Sheldon Whitehouse", "chamber": "Senate", "party": "Democrat", "state": "RI"},
{"name": "John Hickenlooper", "chamber": "Senate", "party": "Democrat", "state": "CO"},
{"name": "Steve Daines", "chamber": "Senate", "party": "Republican", "state": "MT"},
{"name": "Gary Peters", "chamber": "Senate", "party": "Democrat", "state": "MI"},
{"name": "Rick Scott", "chamber": "Senate", "party": "Republican", "state": "FL"},
{"name": "Mark Warner", "chamber": "Senate", "party": "Democrat", "state": "VA"},
{"name": "Dianne Feinstein", "chamber": "Senate", "party": "Democrat", "state": "CA"}, # Note: deceased
# === HOUSE ===
{"name": "Nancy Pelosi", "chamber": "House", "party": "Democrat", "state": "CA"},
{"name": "Brian Higgins", "chamber": "House", "party": "Democrat", "state": "NY"},
{"name": "Michael McCaul", "chamber": "House", "party": "Republican", "state": "TX"},
{"name": "Dan Crenshaw", "chamber": "House", "party": "Republican", "state": "TX"},
{"name": "Marjorie Taylor Greene", "chamber": "House", "party": "Republican", "state": "GA"},
{"name": "Josh Gottheimer", "chamber": "House", "party": "Democrat", "state": "NJ"},
{"name": "Ro Khanna", "chamber": "House", "party": "Democrat", "state": "CA"},
{"name": "Dean Phillips", "chamber": "House", "party": "Democrat", "state": "MN"},
{"name": "Virginia Foxx", "chamber": "House", "party": "Republican", "state": "NC"},
{"name": "Debbie Wasserman Schultz", "chamber": "House", "party": "Democrat", "state": "FL"},
{"name": "Pat Fallon", "chamber": "House", "party": "Republican", "state": "TX"},
{"name": "Kevin Hern", "chamber": "House", "party": "Republican", "state": "OK"},
{"name": "Mark Green", "chamber": "House", "party": "Republican", "state": "TN"},
{"name": "French Hill", "chamber": "House", "party": "Republican", "state": "AR"},
{"name": "John Curtis", "chamber": "House", "party": "Republican", "state": "UT"},
# === HIGH VOLUME TRADERS (Based on 2023-2024 reporting) ===
{"name": "Austin Scott", "chamber": "House", "party": "Republican", "state": "GA"},
{"name": "Nicole Malliotakis", "chamber": "House", "party": "Republican", "state": "NY"},
{"name": "Lois Frankel", "chamber": "House", "party": "Democrat", "state": "FL"},
{"name": "Earl Blumenauer", "chamber": "House", "party": "Democrat", "state": "OR"},
{"name": "Pete Sessions", "chamber": "House", "party": "Republican", "state": "TX"},
]
def save_watchlist(members, filename="watchlist.json"):
"""Save watchlist to file."""
output_path = Path(__file__).parent.parent / "config" / filename
output_path.parent.mkdir(exist_ok=True)
with open(output_path, 'w') as f:
json.dump(members, f, indent=2)
print(f"✅ Saved {len(members)} members to {output_path}")
return output_path
def load_watchlist(filename="watchlist.json"):
"""Load watchlist from file."""
config_path = Path(__file__).parent.parent / "config" / filename
if not config_path.exists():
print(f"⚠️ Watchlist not found at {config_path}")
print(" Creating default watchlist...")
members = get_known_active_traders()
save_watchlist(members, filename)
return members
with open(config_path) as f:
return json.load(f)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Manage Congress member watchlist")
parser.add_argument("--create", action="store_true", help="Create default watchlist")
parser.add_argument("--list", action="store_true", help="List current watchlist")
parser.add_argument("--propublica", action="store_true", help="Fetch from ProPublica API")
args = parser.parse_args()
if args.propublica:
print("Fetching from ProPublica API...")
print("⚠️ You need to set API_KEY in the script first")
print(" Get free key: https://www.propublica.org/datastore/api/propublica-congress-api")
# members = fetch_from_propublica()
# save_watchlist(members)
elif args.create:
members = get_known_active_traders()
save_watchlist(members, "watchlist.json")
print(f"\n✅ Created watchlist with {len(members)} active traders")
elif args.list:
members = load_watchlist()
print(f"\n📋 Watchlist ({len(members)} members):\n")
# Group by chamber
senate = [m for m in members if m['chamber'] == 'Senate']
house = [m for m in members if m['chamber'] == 'House']
print("SENATE:")
for m in sorted(senate, key=lambda x: x['name']):
print(f"{m['name']:30s} ({m['party']:1s}-{m['state']})")
print("\nHOUSE:")
for m in sorted(house, key=lambda x: x['name']):
print(f"{m['name']:30s} ({m['party']:1s}-{m['state']})")
else:
# Default: show known active traders
members = get_known_active_traders()
print(f"\n📋 Known Active Traders ({len(members)} members):\n")
print("These are Congress members with historically high trading activity.\n")
senate = [m for m in members if m['chamber'] == 'Senate']
house = [m for m in members if m['chamber'] == 'House']
print("SENATE:")
for m in sorted(senate, key=lambda x: x['name']):
print(f"{m['name']:30s} ({m['party']:1s}-{m['state']})")
print("\nHOUSE:")
for m in sorted(house, key=lambda x: x['name']):
print(f"{m['name']:30s} ({m['party']:1s}-{m['state']})")
print("\n💡 To create watchlist file: python scripts/fetch_congress_members.py --create")
print("💡 To view saved watchlist: python scripts/fetch_congress_members.py --list")
+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/env python
"""
Generate trading report for watched Congress members.
Shows NEW trades filed recently.
"""
import json
from datetime import date, timedelta
from pathlib import Path
from decimal import Decimal
import click
from sqlalchemy import text
from tabulate import tabulate
from pote.db import get_session
from pote.db.models import Official, Security, Trade
def load_watchlist():
"""Load watchlist of officials to monitor."""
config_path = Path(__file__).parent.parent / "config" / "watchlist.json"
if not config_path.exists():
print("⚠️ No watchlist found. Creating default...")
import subprocess
subprocess.run([
"python",
str(Path(__file__).parent / "fetch_congress_members.py"),
"--create"
])
with open(config_path) as f:
return json.load(f)
def get_new_trades(session, days=7, watchlist=None):
"""
Get trades filed in the last N days.
Args:
session: Database session
days: Look back this many days
watchlist: List of official names to filter (None = all)
"""
since_date = date.today() - timedelta(days=days)
query = text("""
SELECT
o.name,
o.chamber,
o.party,
o.state,
s.ticker,
s.name as company,
s.sector,
t.side,
t.transaction_date,
t.filing_date,
t.value_min,
t.value_max,
t.created_at
FROM trades t
JOIN officials o ON t.official_id = o.id
JOIN securities s ON t.security_id = s.id
WHERE t.created_at >= :since_date
ORDER BY t.created_at DESC, t.transaction_date DESC
""")
result = session.execute(query, {"since_date": since_date})
trades = result.fetchall()
# Filter by watchlist if provided
if watchlist:
watchlist_names = {m['name'].lower() for m in watchlist}
trades = [t for t in trades if t[0].lower() in watchlist_names]
return trades
def format_value(vmin, vmax):
"""Format trade value range."""
if vmax and vmax > vmin:
return f"${float(vmin):,.0f} - ${float(vmax):,.0f}"
else:
return f"${float(vmin):,.0f}+"
def generate_report(trades, format="text"):
"""Generate formatted report."""
if not trades:
return "📭 No new trades found."
if format == "text":
return generate_text_report(trades)
elif format == "html":
return generate_html_report(trades)
elif format == "json":
return generate_json_report(trades)
else:
return generate_text_report(trades)
def generate_text_report(trades):
"""Generate text report."""
report = []
report.append(f"\n{'='*80}")
report.append(f" CONGRESSIONAL TRADING REPORT")
report.append(f" {len(trades)} New Trades")
report.append(f" Generated: {date.today()}")
report.append(f"{'='*80}\n")
# Group by official
by_official = {}
for trade in trades:
name = trade[0]
if name not in by_official:
by_official[name] = []
by_official[name].append(trade)
# Generate section for each official
for official_name, official_trades in by_official.items():
# Header
first_trade = official_trades[0]
chamber, party, state = first_trade[1], first_trade[2], first_trade[3]
report.append(f"\n{''*80}")
report.append(f"👤 {official_name} ({party[0]}-{state}, {chamber})")
report.append(f"{''*80}")
# Trades table
table_data = []
for t in official_trades:
ticker, company, sector = t[4], t[5], t[6]
side, txn_date, filing_date = t[7], t[8], t[9]
vmin, vmax = t[10], t[11]
# Color code side
side_emoji = "🟢 BUY" if side.lower() == "buy" else "🔴 SELL"
table_data.append([
side_emoji,
ticker,
f"{company[:30]}..." if company and len(company) > 30 else (company or ""),
sector or "",
format_value(vmin, vmax),
str(txn_date),
str(filing_date),
])
table = tabulate(
table_data,
headers=["Side", "Ticker", "Company", "Sector", "Value", "Trade Date", "Filed"],
tablefmt="simple"
)
report.append(table)
report.append("")
# Summary statistics
report.append(f"\n{'='*80}")
report.append("📊 SUMMARY")
report.append(f"{'='*80}")
total_buys = sum(1 for t in trades if t[7].lower() == "buy")
total_sells = sum(1 for t in trades if t[7].lower() == "sell")
unique_tickers = len(set(t[4] for t in trades))
unique_officials = len(by_official)
# Top tickers
ticker_counts = {}
for t in trades:
ticker = t[4]
ticker_counts[ticker] = ticker_counts.get(ticker, 0) + 1
top_tickers = sorted(ticker_counts.items(), key=lambda x: x[1], reverse=True)[:5]
report.append(f"\nTotal Trades: {len(trades)}")
report.append(f" Buys: {total_buys}")
report.append(f" Sells: {total_sells}")
report.append(f"Unique Officials: {unique_officials}")
report.append(f"Unique Tickers: {unique_tickers}")
report.append(f"\nTop Tickers:")
for ticker, count in top_tickers:
report.append(f" {ticker:6s} - {count} trades")
report.append(f"\n{'='*80}\n")
return "\n".join(report)
def generate_html_report(trades):
"""Generate HTML email report."""
html = [
"<html><head><style>",
"body { font-family: Arial, sans-serif; }",
"table { border-collapse: collapse; width: 100%; margin: 20px 0; }",
"th { background: #333; color: white; padding: 10px; text-align: left; }",
"td { border: 1px solid #ddd; padding: 8px; }",
".buy { color: green; font-weight: bold; }",
".sell { color: red; font-weight: bold; }",
"</style></head><body>",
f"<h1>Congressional Trading Report</h1>",
f"<p><strong>{len(trades)} New Trades</strong> | Generated: {date.today()}</p>",
]
# Group by official
by_official = {}
for trade in trades:
name = trade[0]
if name not in by_official:
by_official[name] = []
by_official[name].append(trade)
for official_name, official_trades in by_official.items():
first_trade = official_trades[0]
chamber, party, state = first_trade[1], first_trade[2], first_trade[3]
html.append(f"<h2>{official_name} ({party[0]}-{state}, {chamber})</h2>")
html.append("<table>")
html.append("<tr><th>Side</th><th>Ticker</th><th>Company</th><th>Value</th><th>Trade Date</th><th>Filed</th></tr>")
for t in official_trades:
ticker, company, sector = t[4], t[5], t[6]
side, txn_date, filing_date = t[7], t[8], t[9]
vmin, vmax = t[10], t[11]
side_class = "buy" if side.lower() == "buy" else "sell"
side_text = "BUY" if side.lower() == "buy" else "SELL"
html.append(f"<tr>")
html.append(f"<td class='{side_class}'>{side_text}</td>")
html.append(f"<td><strong>{ticker}</strong></td>")
html.append(f"<td>{company or ''}</td>")
html.append(f"<td>{format_value(vmin, vmax)}</td>")
html.append(f"<td>{txn_date}</td>")
html.append(f"<td>{filing_date}</td>")
html.append(f"</tr>")
html.append("</table>")
html.append("</body></html>")
return "\n".join(html)
def generate_json_report(trades):
"""Generate JSON report for programmatic use."""
import json
trades_list = []
for t in trades:
trades_list.append({
"official": t[0],
"chamber": t[1],
"party": t[2],
"state": t[3],
"ticker": t[4],
"company": t[5],
"sector": t[6],
"side": t[7],
"transaction_date": str(t[8]),
"filing_date": str(t[9]),
"value_min": float(t[10]),
"value_max": float(t[11]) if t[11] else None,
})
return json.dumps({
"generated": str(date.today()),
"trade_count": len(trades),
"trades": trades_list
}, indent=2)
@click.command()
@click.option("--days", default=7, help="Look back this many days")
@click.option("--watchlist-only", is_flag=True, help="Only show trades from watchlist")
@click.option("--format", type=click.Choice(["text", "html", "json"]), default="text")
@click.option("--output", help="Output file (default: stdout)")
def main(days, watchlist_only, format, output):
"""Generate trading report for Congress members."""
session = next(get_session())
# Load watchlist if requested
watchlist = None
if watchlist_only:
watchlist = load_watchlist()
print(f"📋 Filtering for {len(watchlist)} officials on watchlist\n")
# Get trades
print(f"🔍 Fetching trades from last {days} days...")
trades = get_new_trades(session, days=days, watchlist=watchlist)
# Generate report
report = generate_report(trades, format=format)
# Output
if output:
Path(output).write_text(report)
print(f"✅ Report saved to {output}")
else:
print(report)
if __name__ == "__main__":
main()
+85
View File
@@ -0,0 +1,85 @@
#!/bin/bash
# Pre-Market-Close POTE Update
# Run at 3 PM ET (1 hour before market close)
# Fetches latest disclosures and generates actionable report
set -e
# --- Configuration ---
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
LOG_DIR="${PROJECT_DIR}/logs"
REPORT_DIR="${PROJECT_DIR}/reports"
LOG_FILE="${LOG_DIR}/pre_market_$(date +%Y%m%d).log"
REPORT_FILE="${REPORT_DIR}/trading_report_$(date +%Y%m%d).txt"
# Ensure directories exist
mkdir -p "$LOG_DIR" "$REPORT_DIR"
# Redirect output to log
exec > >(tee -a "$LOG_FILE") 2>&1
echo "=========================================="
echo " POTE Pre-Market-Close Update"
echo " $(date)"
echo " Running 1 hour before market close"
echo "=========================================="
# Activate virtual environment
cd "$PROJECT_DIR"
source venv/bin/activate
# --- Step 1: Quick Fetch of New Trades ---
echo ""
echo "--- Fetching Latest Congressional Trades ---"
python scripts/fetch_congressional_trades.py --days 3
FETCH_EXIT=$?
if [ $FETCH_EXIT -ne 0 ]; then
echo "⚠️ WARNING: Failed to fetch trades (API may be down)"
echo " Generating report from existing data..."
fi
# --- Step 2: Quick Security Enrichment ---
echo ""
echo "--- Enriching New Securities ---"
python scripts/enrich_securities.py --limit 10
ENRICH_EXIT=$?
# --- Step 3: Generate Trading Report ---
echo ""
echo "--- Generating Trading Report ---"
python scripts/generate_trading_report.py \
--days 7 \
--watchlist-only \
--format text \
--output "$REPORT_FILE"
REPORT_EXIT=$?
if [ $REPORT_EXIT -eq 0 ]; then
echo "✅ Report saved to: $REPORT_FILE"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "📊 REPORT PREVIEW"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
cat "$REPORT_FILE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "❌ Failed to generate report"
fi
# --- Step 4: Optional Price Update (Quick) ---
# Uncomment if you want prices updated before market close
# echo ""
# echo "--- Quick Price Update ---"
# python scripts/fetch_sample_prices.py --days 5
echo ""
echo "=========================================="
echo " Update Complete - $(date)"
echo "=========================================="
# Exit successfully even if some steps warned
exit 0