Add data update tools and Phase 2 plan
- scripts/add_custom_trades.py: Manual trade entry - scripts/scrape_alternative_sources.py: CSV import - scripts/daily_update.sh: Automated daily updates - docs/09_data_updates.md: Complete update guide - docs/PR4_PLAN.md: Phase 2 analytics plan Enables users to add representatives and set up auto-updates
This commit is contained in:
ilia
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Manually add trades for specific representatives.
|
||||
Useful when you want to track specific officials or add data from other sources.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
from pote.db import get_session
|
||||
from pote.db.models import Official, Security, Trade
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def add_trade(
|
||||
session,
|
||||
official_name: str,
|
||||
party: str,
|
||||
chamber: str,
|
||||
state: str,
|
||||
ticker: str,
|
||||
company_name: str,
|
||||
side: str,
|
||||
value_min: float,
|
||||
value_max: float,
|
||||
transaction_date: str, # YYYY-MM-DD
|
||||
disclosure_date: str | None = None,
|
||||
):
|
||||
"""Add a single trade to the database."""
|
||||
|
||||
# Get or create official
|
||||
official = session.query(Official).filter_by(name=official_name).first()
|
||||
if not official:
|
||||
official = Official(
|
||||
name=official_name,
|
||||
party=party,
|
||||
chamber=chamber,
|
||||
state=state,
|
||||
)
|
||||
session.add(official)
|
||||
session.flush()
|
||||
logger.info(f"Created official: {official_name}")
|
||||
|
||||
# Get or create security
|
||||
security = session.query(Security).filter_by(ticker=ticker).first()
|
||||
if not security:
|
||||
security = Security(ticker=ticker, name=company_name)
|
||||
session.add(security)
|
||||
session.flush()
|
||||
logger.info(f"Created security: {ticker}")
|
||||
|
||||
# Create trade
|
||||
trade = Trade(
|
||||
official_id=official.id,
|
||||
security_id=security.id,
|
||||
source="manual",
|
||||
transaction_date=datetime.strptime(transaction_date, "%Y-%m-%d").date(),
|
||||
filing_date=datetime.strptime(disclosure_date, "%Y-%m-%d").date() if disclosure_date else None,
|
||||
side=side,
|
||||
value_min=Decimal(str(value_min)),
|
||||
value_max=Decimal(str(value_max)),
|
||||
)
|
||||
session.add(trade)
|
||||
logger.info(f"Added trade: {official_name} {side} {ticker}")
|
||||
|
||||
return trade
|
||||
|
||||
|
||||
def main():
|
||||
"""Example: Add some trades manually."""
|
||||
|
||||
with next(get_session()) as session:
|
||||
# Example: Add trades for Elizabeth Warren
|
||||
logger.info("Adding trades for Elizabeth Warren...")
|
||||
|
||||
add_trade(
|
||||
session,
|
||||
official_name="Elizabeth Warren",
|
||||
party="Democrat",
|
||||
chamber="Senate",
|
||||
state="MA",
|
||||
ticker="AMZN",
|
||||
company_name="Amazon.com Inc.",
|
||||
side="sell",
|
||||
value_min=15001,
|
||||
value_max=50000,
|
||||
transaction_date="2024-11-15",
|
||||
disclosure_date="2024-12-01",
|
||||
)
|
||||
|
||||
add_trade(
|
||||
session,
|
||||
official_name="Elizabeth Warren",
|
||||
party="Democrat",
|
||||
chamber="Senate",
|
||||
state="MA",
|
||||
ticker="META",
|
||||
company_name="Meta Platforms Inc.",
|
||||
side="sell",
|
||||
value_min=50001,
|
||||
value_max=100000,
|
||||
transaction_date="2024-11-20",
|
||||
disclosure_date="2024-12-05",
|
||||
)
|
||||
|
||||
# Example: Add trades for Mitt Romney
|
||||
logger.info("Adding trades for Mitt Romney...")
|
||||
|
||||
add_trade(
|
||||
session,
|
||||
official_name="Mitt Romney",
|
||||
party="Republican",
|
||||
chamber="Senate",
|
||||
state="UT",
|
||||
ticker="BRK.B",
|
||||
company_name="Berkshire Hathaway Inc.",
|
||||
side="buy",
|
||||
value_min=100001,
|
||||
value_max=250000,
|
||||
transaction_date="2024-10-01",
|
||||
disclosure_date="2024-10-15",
|
||||
)
|
||||
|
||||
session.commit()
|
||||
logger.info("✅ All trades added successfully!")
|
||||
|
||||
# Show summary
|
||||
from sqlalchemy import text
|
||||
result = session.execute(text("""
|
||||
SELECT o.name, COUNT(t.id) as trade_count
|
||||
FROM officials o
|
||||
LEFT JOIN trades t ON o.id = t.official_id
|
||||
GROUP BY o.name
|
||||
ORDER BY trade_count DESC
|
||||
"""))
|
||||
|
||||
print("\n=== Officials Summary ===")
|
||||
for row in result:
|
||||
print(f" {row[0]:25s} - {row[1]} trades")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
# Daily update script for POTE
|
||||
# Run this via cron to automatically fetch new data
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
POTE_DIR="/home/poteapp/pote"
|
||||
LOG_DIR="/home/poteapp/logs"
|
||||
LOG_FILE="$LOG_DIR/daily_update_$(date +%Y%m%d).log"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
echo "=== POTE Daily Update: $(date) ===" | tee -a "$LOG_FILE"
|
||||
|
||||
cd "$POTE_DIR"
|
||||
source venv/bin/activate
|
||||
|
||||
# 1. Fetch new congressional trades (if House Stock Watcher is back up)
|
||||
echo "[1/4] Fetching congressional trades..." | tee -a "$LOG_FILE"
|
||||
if python scripts/fetch_congressional_trades.py --days 7 >> "$LOG_FILE" 2>&1; then
|
||||
echo "✓ Trades fetched successfully" | tee -a "$LOG_FILE"
|
||||
else
|
||||
echo "✗ Trade fetch failed (API might be down)" | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# 2. Enrich any new securities
|
||||
echo "[2/4] Enriching securities..." | tee -a "$LOG_FILE"
|
||||
if python scripts/enrich_securities.py >> "$LOG_FILE" 2>&1; then
|
||||
echo "✓ Securities enriched" | tee -a "$LOG_FILE"
|
||||
else
|
||||
echo "✗ Security enrichment failed" | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# 3. Update prices for all securities
|
||||
echo "[3/4] Fetching price data..." | tee -a "$LOG_FILE"
|
||||
if python scripts/fetch_sample_prices.py >> "$LOG_FILE" 2>&1; then
|
||||
echo "✓ Prices updated" | tee -a "$LOG_FILE"
|
||||
else
|
||||
echo "✗ Price fetch failed" | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# 4. Generate summary
|
||||
echo "[4/4] Generating summary..." | tee -a "$LOG_FILE"
|
||||
python << 'EOF' | tee -a "$LOG_FILE"
|
||||
from sqlalchemy import text
|
||||
from pote.db import engine
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
with engine.connect() as conn:
|
||||
# Get counts
|
||||
officials = conn.execute(text("SELECT COUNT(*) FROM officials")).scalar()
|
||||
trades = conn.execute(text("SELECT COUNT(*) FROM trades")).scalar()
|
||||
securities = conn.execute(text("SELECT COUNT(*) FROM securities")).scalar()
|
||||
|
||||
# Get new trades in last 7 days
|
||||
week_ago = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
|
||||
new_trades = conn.execute(
|
||||
text(f"SELECT COUNT(*) FROM trades WHERE created_at >= '{week_ago}'")
|
||||
).scalar()
|
||||
|
||||
print(f"\n📊 Database Summary:")
|
||||
print(f" Officials: {officials:,}")
|
||||
print(f" Securities: {securities:,}")
|
||||
print(f" Trades: {trades:,}")
|
||||
print(f" New (7d): {new_trades:,}")
|
||||
EOF
|
||||
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "=== Update Complete: $(date) ===" | tee -a "$LOG_FILE"
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
|
||||
# Keep only last 30 days of logs
|
||||
find "$LOG_DIR" -name "daily_update_*.log" -mtime +30 -delete
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scrape congressional trades from alternative sources.
|
||||
Options:
|
||||
1. Senate Stock Watcher (if available)
|
||||
2. QuiverQuant (requires API key)
|
||||
3. Capitol Trades (web scraping - be careful)
|
||||
4. Manual CSV import
|
||||
"""
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pote.db import get_session
|
||||
from pote.ingestion.trade_loader import TradeLoader
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def import_from_csv(csv_path: str):
|
||||
"""
|
||||
Import trades from CSV file.
|
||||
|
||||
CSV format:
|
||||
name,party,chamber,state,ticker,side,value_min,value_max,transaction_date,disclosure_date
|
||||
"""
|
||||
|
||||
logger.info(f"Reading trades from {csv_path}")
|
||||
|
||||
with open(csv_path, 'r') as f:
|
||||
reader = csv.DictReader(f)
|
||||
transactions = []
|
||||
|
||||
for row in reader:
|
||||
# Convert CSV row to transaction format
|
||||
txn = {
|
||||
"representative": row["name"],
|
||||
"party": row["party"],
|
||||
"house": row["chamber"], # "House" or "Senate"
|
||||
"state": row.get("state", ""),
|
||||
"district": row.get("district", ""),
|
||||
"ticker": row["ticker"],
|
||||
"transaction": row["side"].capitalize(), # "Purchase" or "Sale"
|
||||
"amount": f"${row['value_min']} - ${row['value_max']}",
|
||||
"transaction_date": row["transaction_date"],
|
||||
"disclosure_date": row.get("disclosure_date", row["transaction_date"]),
|
||||
}
|
||||
transactions.append(txn)
|
||||
|
||||
logger.info(f"Loaded {len(transactions)} transactions from CSV")
|
||||
|
||||
# Ingest into database
|
||||
with next(get_session()) as session:
|
||||
loader = TradeLoader(session)
|
||||
stats = loader.ingest_transactions(transactions, source="csv_import")
|
||||
|
||||
logger.info(f"✅ Ingested: {stats['officials_created']} officials, "
|
||||
f"{stats['securities_created']} securities, "
|
||||
f"{stats['trades_ingested']} trades")
|
||||
|
||||
|
||||
def create_sample_csv(output_path: str = "trades_template.csv"):
|
||||
"""Create a template CSV file for manual entry."""
|
||||
|
||||
template_data = [
|
||||
{
|
||||
"name": "Bernie Sanders",
|
||||
"party": "Independent",
|
||||
"chamber": "Senate",
|
||||
"state": "VT",
|
||||
"district": "",
|
||||
"ticker": "COIN",
|
||||
"side": "sell",
|
||||
"value_min": "15001",
|
||||
"value_max": "50000",
|
||||
"transaction_date": "2024-12-01",
|
||||
"disclosure_date": "2024-12-15",
|
||||
},
|
||||
{
|
||||
"name": "Alexandria Ocasio-Cortez",
|
||||
"party": "Democrat",
|
||||
"chamber": "House",
|
||||
"state": "NY",
|
||||
"district": "NY-14",
|
||||
"ticker": "PLTR",
|
||||
"side": "buy",
|
||||
"value_min": "1001",
|
||||
"value_max": "15000",
|
||||
"transaction_date": "2024-11-15",
|
||||
"disclosure_date": "2024-12-01",
|
||||
},
|
||||
]
|
||||
|
||||
with open(output_path, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=template_data[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(template_data)
|
||||
|
||||
logger.info(f"✅ Created template CSV: {output_path}")
|
||||
logger.info("Edit this file and run: python scripts/scrape_alternative_sources.py import <file>")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage:")
|
||||
print(" python scripts/scrape_alternative_sources.py template # Create CSV template")
|
||||
print(" python scripts/scrape_alternative_sources.py import <csv_file> # Import from CSV")
|
||||
sys.exit(1)
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "template":
|
||||
create_sample_csv()
|
||||
elif command == "import":
|
||||
if len(sys.argv) < 3:
|
||||
print("Error: Please specify CSV file to import")
|
||||
sys.exit(1)
|
||||
import_from_csv(sys.argv[2])
|
||||
else:
|
||||
print(f"Unknown command: {command}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user