Add comprehensive automation system
New Scripts: - scripts/daily_fetch.sh: Automated daily data updates * Fetches congressional trades (last 7 days) * Enriches securities (name, sector, industry) * Updates price data for all securities * Calculates returns and metrics * Logs everything to logs/ directory - scripts/setup_automation.sh: Interactive automation setup * Makes scripts executable * Creates log directories * Configures cron jobs (multiple schedule options) * Guides user through setup Documentation: - docs/10_automation.md: Complete automation guide * Explains disclosure timing (30-45 day legal lag) * Why daily updates are optimal (not hourly/real-time) * Cron job setup instructions * Systemd timer alternative * Email notifications (optional) * Monitoring and logging * Failure handling * Performance optimization Key Insights: ❌ No real-time data possible (STOCK Act = 30-45 day lag) ✅ Daily updates are optimal ✅ Automated via cron jobs ✅ Handles API failures gracefully ✅ Logs everything for debugging
This commit is contained in:
ilia
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/bin/bash
|
||||
# Daily POTE Data Update Script
|
||||
# Run this once per day to fetch new trades and prices
|
||||
# Recommended: 7 AM daily (after markets close and disclosures are filed)
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
# --- Configuration ---
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
LOG_DIR="${PROJECT_DIR}/logs"
|
||||
LOG_FILE="${LOG_DIR}/daily_fetch_$(date +%Y%m%d).log"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Redirect all output to log file
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
|
||||
echo "=========================================="
|
||||
echo " POTE Daily Data Fetch"
|
||||
echo " $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# Activate virtual environment
|
||||
cd "$PROJECT_DIR"
|
||||
source venv/bin/activate
|
||||
|
||||
# --- Step 1: Fetch Congressional Trades ---
|
||||
echo ""
|
||||
echo "--- Step 1: Fetching Congressional Trades ---"
|
||||
# Fetch last 7 days (to catch any late filings)
|
||||
python scripts/fetch_congressional_trades.py --days 7
|
||||
TRADES_EXIT=$?
|
||||
|
||||
if [ $TRADES_EXIT -ne 0 ]; then
|
||||
echo "⚠️ WARNING: Failed to fetch congressional trades"
|
||||
echo " This is likely because House Stock Watcher API is down"
|
||||
echo " Continuing with other steps..."
|
||||
fi
|
||||
|
||||
# --- Step 2: Enrich Securities ---
|
||||
echo ""
|
||||
echo "--- Step 2: Enriching Securities ---"
|
||||
# Add company names, sectors, industries for any new tickers
|
||||
python scripts/enrich_securities.py
|
||||
ENRICH_EXIT=$?
|
||||
|
||||
if [ $ENRICH_EXIT -ne 0 ]; then
|
||||
echo "⚠️ WARNING: Failed to enrich securities"
|
||||
fi
|
||||
|
||||
# --- Step 3: Fetch Price Data ---
|
||||
echo ""
|
||||
echo "--- Step 3: Fetching Price Data ---"
|
||||
# Fetch prices for all securities
|
||||
python scripts/fetch_sample_prices.py
|
||||
PRICES_EXIT=$?
|
||||
|
||||
if [ $PRICES_EXIT -ne 0 ]; then
|
||||
echo "⚠️ WARNING: Failed to fetch price data"
|
||||
fi
|
||||
|
||||
# --- Step 4: Calculate Returns (Optional) ---
|
||||
echo ""
|
||||
echo "--- Step 4: Calculating Returns ---"
|
||||
python scripts/calculate_all_returns.py --window 90 --limit 100
|
||||
CALC_EXIT=$?
|
||||
|
||||
if [ $CALC_EXIT -ne 0 ]; then
|
||||
echo "⚠️ WARNING: Failed to calculate returns"
|
||||
fi
|
||||
|
||||
# --- Summary ---
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Daily Fetch Complete"
|
||||
echo " $(date)"
|
||||
echo "=========================================="
|
||||
|
||||
# Show quick stats
|
||||
python << 'PYEOF'
|
||||
from sqlalchemy import text
|
||||
from pote.db import engine
|
||||
from datetime import datetime
|
||||
|
||||
print("\n📊 Current Database Stats:")
|
||||
with engine.connect() as conn:
|
||||
officials = conn.execute(text("SELECT COUNT(*) FROM officials")).scalar()
|
||||
trades = conn.execute(text("SELECT COUNT(*) FROM trades")).scalar()
|
||||
securities = conn.execute(text("SELECT COUNT(*) FROM securities")).scalar()
|
||||
prices = conn.execute(text("SELECT COUNT(*) FROM prices")).scalar()
|
||||
|
||||
print(f" Officials: {officials:,}")
|
||||
print(f" Securities: {securities:,}")
|
||||
print(f" Trades: {trades:,}")
|
||||
print(f" Prices: {prices:,}")
|
||||
|
||||
# Show most recent trade
|
||||
result = conn.execute(text("""
|
||||
SELECT o.name, s.ticker, t.side, t.transaction_date
|
||||
FROM trades t
|
||||
JOIN officials o ON t.official_id = o.id
|
||||
JOIN securities s ON t.security_id = s.id
|
||||
ORDER BY t.transaction_date DESC
|
||||
LIMIT 1
|
||||
""")).fetchone()
|
||||
|
||||
if result:
|
||||
print(f"\n📈 Most Recent Trade:")
|
||||
print(f" {result[0]} - {result[2].upper()} {result[1]} on {result[3]}")
|
||||
|
||||
print()
|
||||
PYEOF
|
||||
|
||||
# Exit with success (even if some steps warned)
|
||||
exit 0
|
||||
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/bin/bash
|
||||
# Setup Automation for POTE
|
||||
# Run this once on your Proxmox container to enable daily updates
|
||||
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo " POTE Automation Setup"
|
||||
echo "=========================================="
|
||||
|
||||
# Detect if we're root or regular user
|
||||
if [ "$EUID" -eq 0 ]; then
|
||||
echo "⚠️ Running as root. Will setup for poteapp user."
|
||||
TARGET_USER="poteapp"
|
||||
TARGET_HOME="/home/poteapp"
|
||||
else
|
||||
TARGET_USER="$USER"
|
||||
TARGET_HOME="$HOME"
|
||||
fi
|
||||
|
||||
POTE_DIR="${TARGET_HOME}/pote"
|
||||
|
||||
# Check if POTE directory exists
|
||||
if [ ! -d "$POTE_DIR" ]; then
|
||||
echo "❌ Error: POTE directory not found at $POTE_DIR"
|
||||
echo " Please clone the repository first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Found POTE at: $POTE_DIR"
|
||||
|
||||
# Make scripts executable
|
||||
echo ""
|
||||
echo "Making scripts executable..."
|
||||
chmod +x "${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
chmod +x "${POTE_DIR}/scripts/fetch_congressional_trades.py"
|
||||
chmod +x "${POTE_DIR}/scripts/enrich_securities.py"
|
||||
chmod +x "${POTE_DIR}/scripts/fetch_sample_prices.py"
|
||||
|
||||
# Create logs directory
|
||||
echo "Creating logs directory..."
|
||||
mkdir -p "${POTE_DIR}/logs"
|
||||
|
||||
# Test the daily fetch script
|
||||
echo ""
|
||||
echo "Testing daily fetch script (dry run)..."
|
||||
echo "This may take a few minutes..."
|
||||
cd "$POTE_DIR"
|
||||
|
||||
if [ "$EUID" -eq 0 ]; then
|
||||
su - $TARGET_USER -c "cd ${POTE_DIR} && source venv/bin/activate && python --version"
|
||||
else
|
||||
source venv/bin/activate
|
||||
python --version
|
||||
fi
|
||||
|
||||
# Setup cron job
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Cron Job Setup"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Choose schedule:"
|
||||
echo " 1) Daily at 7 AM (recommended)"
|
||||
echo " 2) Twice daily (7 AM and 7 PM)"
|
||||
echo " 3) Weekdays only at 7 AM"
|
||||
echo " 4) Custom (I'll help you configure)"
|
||||
echo " 5) Skip (manual setup)"
|
||||
echo ""
|
||||
read -p "Enter choice [1-5]: " choice
|
||||
|
||||
CRON_LINE=""
|
||||
|
||||
case $choice in
|
||||
1)
|
||||
CRON_LINE="0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
;;
|
||||
2)
|
||||
CRON_LINE="0 7,19 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
;;
|
||||
3)
|
||||
CRON_LINE="0 7 * * 1-5 ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
;;
|
||||
4)
|
||||
echo ""
|
||||
echo "Cron format: MIN HOUR DAY MONTH WEEKDAY"
|
||||
echo "Examples:"
|
||||
echo " 0 7 * * * = Daily at 7 AM"
|
||||
echo " 0 */6 * * * = Every 6 hours"
|
||||
echo " 0 0 * * 0 = Weekly on Sunday"
|
||||
read -p "Enter cron schedule: " custom_schedule
|
||||
CRON_LINE="${custom_schedule} ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
;;
|
||||
5)
|
||||
echo "Skipping cron setup. You can add manually with:"
|
||||
echo " crontab -e"
|
||||
echo " Add: 0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
CRON_LINE=""
|
||||
;;
|
||||
*)
|
||||
echo "Invalid choice. Skipping cron setup."
|
||||
CRON_LINE=""
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -n "$CRON_LINE" ]; then
|
||||
echo ""
|
||||
echo "Adding to crontab: $CRON_LINE"
|
||||
|
||||
if [ "$EUID" -eq 0 ]; then
|
||||
# Add as target user
|
||||
(su - $TARGET_USER -c "crontab -l" 2>/dev/null || true; echo "$CRON_LINE") | \
|
||||
su - $TARGET_USER -c "crontab -"
|
||||
else
|
||||
# Add as current user
|
||||
(crontab -l 2>/dev/null || true; echo "$CRON_LINE") | crontab -
|
||||
fi
|
||||
|
||||
echo "✅ Cron job added!"
|
||||
echo ""
|
||||
echo "View with: crontab -l"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " Setup Complete!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "📝 What was configured:"
|
||||
echo " ✅ Scripts made executable"
|
||||
echo " ✅ Logs directory created: ${POTE_DIR}/logs"
|
||||
if [ -n "$CRON_LINE" ]; then
|
||||
echo " ✅ Cron job scheduled"
|
||||
fi
|
||||
echo ""
|
||||
echo "🧪 Test manually:"
|
||||
echo " ${POTE_DIR}/scripts/daily_fetch.sh"
|
||||
echo ""
|
||||
echo "📊 View logs:"
|
||||
echo " tail -f ${POTE_DIR}/logs/daily_fetch_\$(date +%Y%m%d).log"
|
||||
echo ""
|
||||
echo "⚙️ Manage cron:"
|
||||
echo " crontab -l # View cron jobs"
|
||||
echo " crontab -e # Edit cron jobs"
|
||||
echo ""
|
||||
echo "📚 Documentation:"
|
||||
echo " ${POTE_DIR}/docs/10_automation.md"
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user