POTE/scripts/daily_fetch.sh

#!/bin/bash
# Daily POTE Data Update Script
# Run this once per day to fetch new trades and prices
# Recommended: 7 AM daily (after markets close and disclosures are filed)

set -e  # Exit on error

# --- Configuration ---
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
LOG_DIR="${PROJECT_DIR}/logs"
LOG_FILE="${LOG_DIR}/daily_fetch_$(date +%Y%m%d).log"

# Ensure log directory exists
mkdir -p "$LOG_DIR"

# Redirect all output to log file
exec > >(tee -a "$LOG_FILE") 2>&1

echo "=========================================="
echo "  POTE Daily Data Fetch"
echo "  $(date)"
echo "=========================================="

# Activate virtual environment
cd "$PROJECT_DIR"
source venv/bin/activate

# --- Step 1: Fetch Congressional Trades ---
echo ""
echo "--- Step 1: Fetching Congressional Trades ---"
# Fetch last 7 days (to catch any late filings)
python scripts/fetch_congressional_trades.py --days 7
TRADES_EXIT=$?

if [ $TRADES_EXIT -ne 0 ]; then
    echo "⚠️  WARNING: Failed to fetch congressional trades"
    echo "   This is likely because House Stock Watcher API is down"
    echo "   Continuing with other steps..."
fi

# --- Step 2: Enrich Securities ---
echo ""
echo "--- Step 2: Enriching Securities ---"
# Add company names, sectors, industries for any new tickers
python scripts/enrich_securities.py
ENRICH_EXIT=$?

if [ $ENRICH_EXIT -ne 0 ]; then
    echo "⚠️  WARNING: Failed to enrich securities"
fi

# --- Step 3: Fetch Price Data ---
echo ""
echo "--- Step 3: Fetching Price Data ---"
# Fetch prices for all securities
python scripts/fetch_sample_prices.py
PRICES_EXIT=$?

if [ $PRICES_EXIT -ne 0 ]; then
    echo "⚠️  WARNING: Failed to fetch price data"
fi

# --- Step 4: Calculate Returns (Optional) ---
echo ""
echo "--- Step 4: Calculating Returns ---"
python scripts/calculate_all_returns.py --window 90 --limit 100
CALC_EXIT=$?

if [ $CALC_EXIT -ne 0 ]; then
    echo "⚠️  WARNING: Failed to calculate returns"
fi

# --- Summary ---
echo ""
echo "=========================================="
echo "  Daily Fetch Complete"
echo "  $(date)"
echo "=========================================="

# Show quick stats
python << 'PYEOF'
from sqlalchemy import text
from pote.db import engine
from datetime import datetime

print("\n📊 Current Database Stats:")
with engine.connect() as conn:
    officials = conn.execute(text("SELECT COUNT(*) FROM officials")).scalar()
    trades = conn.execute(text("SELECT COUNT(*) FROM trades")).scalar()
    securities = conn.execute(text("SELECT COUNT(*) FROM securities")).scalar()
    prices = conn.execute(text("SELECT COUNT(*) FROM prices")).scalar()
    
    print(f"   Officials:  {officials:,}")
    print(f"   Securities: {securities:,}")
    print(f"   Trades:     {trades:,}")
    print(f"   Prices:     {prices:,}")
    
    # Show most recent trade
    result = conn.execute(text("""
        SELECT o.name, s.ticker, t.side, t.transaction_date
        FROM trades t
        JOIN officials o ON t.official_id = o.id
        JOIN securities s ON t.security_id = s.id
        ORDER BY t.transaction_date DESC
        LIMIT 1
    """)).fetchone()
    
    if result:
        print(f"\n📈 Most Recent Trade:")
        print(f"   {result[0]} - {result[2].upper()} {result[1]} on {result[3]}")

print()
PYEOF

# Exit with success (even if some steps warned)
exit 0