3a89c1e6d2
New Scripts: - scripts/daily_fetch.sh: Automated daily data updates * Fetches congressional trades (last 7 days) * Enriches securities (name, sector, industry) * Updates price data for all securities * Calculates returns and metrics * Logs everything to logs/ directory - scripts/setup_automation.sh: Interactive automation setup * Makes scripts executable * Creates log directories * Configures cron jobs (multiple schedule options) * Guides user through setup Documentation: - docs/10_automation.md: Complete automation guide * Explains disclosure timing (30-45 day legal lag) * Why daily updates are optimal (not hourly/real-time) * Cron job setup instructions * Systemd timer alternative * Email notifications (optional) * Monitoring and logging * Failure handling * Performance optimization Key Insights: ❌ No real-time data possible (STOCK Act = 30-45 day lag) ✅ Daily updates are optimal ✅ Automated via cron jobs ✅ Handles API failures gracefully ✅ Logs everything for debugging
119 lines
3.3 KiB
Bash
Executable File
119 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Daily POTE Data Update Script
|
|
# Run this once per day to fetch new trades and prices
|
|
# Recommended: 7 AM daily (after markets close and disclosures are filed)
|
|
|
|
set -e # Exit on error
|
|
|
|
# --- Configuration ---
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
LOG_DIR="${PROJECT_DIR}/logs"
|
|
LOG_FILE="${LOG_DIR}/daily_fetch_$(date +%Y%m%d).log"
|
|
|
|
# Ensure log directory exists
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Redirect all output to log file
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
|
|
echo "=========================================="
|
|
echo " POTE Daily Data Fetch"
|
|
echo " $(date)"
|
|
echo "=========================================="
|
|
|
|
# Activate virtual environment
|
|
cd "$PROJECT_DIR"
|
|
source venv/bin/activate
|
|
|
|
# --- Step 1: Fetch Congressional Trades ---
|
|
echo ""
|
|
echo "--- Step 1: Fetching Congressional Trades ---"
|
|
# Fetch last 7 days (to catch any late filings)
|
|
python scripts/fetch_congressional_trades.py --days 7
|
|
TRADES_EXIT=$?
|
|
|
|
if [ $TRADES_EXIT -ne 0 ]; then
|
|
echo "⚠️ WARNING: Failed to fetch congressional trades"
|
|
echo " This is likely because House Stock Watcher API is down"
|
|
echo " Continuing with other steps..."
|
|
fi
|
|
|
|
# --- Step 2: Enrich Securities ---
|
|
echo ""
|
|
echo "--- Step 2: Enriching Securities ---"
|
|
# Add company names, sectors, industries for any new tickers
|
|
python scripts/enrich_securities.py
|
|
ENRICH_EXIT=$?
|
|
|
|
if [ $ENRICH_EXIT -ne 0 ]; then
|
|
echo "⚠️ WARNING: Failed to enrich securities"
|
|
fi
|
|
|
|
# --- Step 3: Fetch Price Data ---
|
|
echo ""
|
|
echo "--- Step 3: Fetching Price Data ---"
|
|
# Fetch prices for all securities
|
|
python scripts/fetch_sample_prices.py
|
|
PRICES_EXIT=$?
|
|
|
|
if [ $PRICES_EXIT -ne 0 ]; then
|
|
echo "⚠️ WARNING: Failed to fetch price data"
|
|
fi
|
|
|
|
# --- Step 4: Calculate Returns (Optional) ---
|
|
echo ""
|
|
echo "--- Step 4: Calculating Returns ---"
|
|
python scripts/calculate_all_returns.py --window 90 --limit 100
|
|
CALC_EXIT=$?
|
|
|
|
if [ $CALC_EXIT -ne 0 ]; then
|
|
echo "⚠️ WARNING: Failed to calculate returns"
|
|
fi
|
|
|
|
# --- Summary ---
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Daily Fetch Complete"
|
|
echo " $(date)"
|
|
echo "=========================================="
|
|
|
|
# Show quick stats
|
|
python << 'PYEOF'
|
|
from sqlalchemy import text
|
|
from pote.db import engine
|
|
from datetime import datetime
|
|
|
|
print("\n📊 Current Database Stats:")
|
|
with engine.connect() as conn:
|
|
officials = conn.execute(text("SELECT COUNT(*) FROM officials")).scalar()
|
|
trades = conn.execute(text("SELECT COUNT(*) FROM trades")).scalar()
|
|
securities = conn.execute(text("SELECT COUNT(*) FROM securities")).scalar()
|
|
prices = conn.execute(text("SELECT COUNT(*) FROM prices")).scalar()
|
|
|
|
print(f" Officials: {officials:,}")
|
|
print(f" Securities: {securities:,}")
|
|
print(f" Trades: {trades:,}")
|
|
print(f" Prices: {prices:,}")
|
|
|
|
# Show most recent trade
|
|
result = conn.execute(text("""
|
|
SELECT o.name, s.ticker, t.side, t.transaction_date
|
|
FROM trades t
|
|
JOIN officials o ON t.official_id = o.id
|
|
JOIN securities s ON t.security_id = s.id
|
|
ORDER BY t.transaction_date DESC
|
|
LIMIT 1
|
|
""")).fetchone()
|
|
|
|
if result:
|
|
print(f"\n📈 Most Recent Trade:")
|
|
print(f" {result[0]} - {result[2].upper()} {result[1]} on {result[3]}")
|
|
|
|
print()
|
|
PYEOF
|
|
|
|
# Exit with success (even if some steps warned)
|
|
exit 0
|
|
|