New Scripts: - scripts/daily_fetch.sh: Automated daily data updates * Fetches congressional trades (last 7 days) * Enriches securities (name, sector, industry) * Updates price data for all securities * Calculates returns and metrics * Logs everything to logs/ directory - scripts/setup_automation.sh: Interactive automation setup * Makes scripts executable * Creates log directories * Configures cron jobs (multiple schedule options) * Guides user through setup Documentation: - docs/10_automation.md: Complete automation guide * Explains disclosure timing (30-45 day legal lag) * Why daily updates are optimal (not hourly/real-time) * Cron job setup instructions * Systemd timer alternative * Email notifications (optional) * Monitoring and logging * Failure handling * Performance optimization Key Insights: ❌ No real-time data possible (STOCK Act = 30-45 day lag) ✅ Daily updates are optimal ✅ Automated via cron jobs ✅ Handles API failures gracefully ✅ Logs everything for debugging
151 lines
3.9 KiB
Bash
Executable File
151 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Setup Automation for POTE
|
|
# Run this once on your Proxmox container to enable daily updates
|
|
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo " POTE Automation Setup"
|
|
echo "=========================================="
|
|
|
|
# Detect if we're root or regular user
|
|
if [ "$EUID" -eq 0 ]; then
|
|
echo "⚠️ Running as root. Will setup for poteapp user."
|
|
TARGET_USER="poteapp"
|
|
TARGET_HOME="/home/poteapp"
|
|
else
|
|
TARGET_USER="$USER"
|
|
TARGET_HOME="$HOME"
|
|
fi
|
|
|
|
POTE_DIR="${TARGET_HOME}/pote"
|
|
|
|
# Check if POTE directory exists
|
|
if [ ! -d "$POTE_DIR" ]; then
|
|
echo "❌ Error: POTE directory not found at $POTE_DIR"
|
|
echo " Please clone the repository first."
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Found POTE at: $POTE_DIR"
|
|
|
|
# Make scripts executable
|
|
echo ""
|
|
echo "Making scripts executable..."
|
|
chmod +x "${POTE_DIR}/scripts/daily_fetch.sh"
|
|
chmod +x "${POTE_DIR}/scripts/fetch_congressional_trades.py"
|
|
chmod +x "${POTE_DIR}/scripts/enrich_securities.py"
|
|
chmod +x "${POTE_DIR}/scripts/fetch_sample_prices.py"
|
|
|
|
# Create logs directory
|
|
echo "Creating logs directory..."
|
|
mkdir -p "${POTE_DIR}/logs"
|
|
|
|
# Test the daily fetch script
|
|
echo ""
|
|
echo "Testing daily fetch script (dry run)..."
|
|
echo "This may take a few minutes..."
|
|
cd "$POTE_DIR"
|
|
|
|
if [ "$EUID" -eq 0 ]; then
|
|
su - $TARGET_USER -c "cd ${POTE_DIR} && source venv/bin/activate && python --version"
|
|
else
|
|
source venv/bin/activate
|
|
python --version
|
|
fi
|
|
|
|
# Setup cron job
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Cron Job Setup"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Choose schedule:"
|
|
echo " 1) Daily at 7 AM (recommended)"
|
|
echo " 2) Twice daily (7 AM and 7 PM)"
|
|
echo " 3) Weekdays only at 7 AM"
|
|
echo " 4) Custom (I'll help you configure)"
|
|
echo " 5) Skip (manual setup)"
|
|
echo ""
|
|
read -p "Enter choice [1-5]: " choice
|
|
|
|
CRON_LINE=""
|
|
|
|
case $choice in
|
|
1)
|
|
CRON_LINE="0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
2)
|
|
CRON_LINE="0 7,19 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
3)
|
|
CRON_LINE="0 7 * * 1-5 ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
4)
|
|
echo ""
|
|
echo "Cron format: MIN HOUR DAY MONTH WEEKDAY"
|
|
echo "Examples:"
|
|
echo " 0 7 * * * = Daily at 7 AM"
|
|
echo " 0 */6 * * * = Every 6 hours"
|
|
echo " 0 0 * * 0 = Weekly on Sunday"
|
|
read -p "Enter cron schedule: " custom_schedule
|
|
CRON_LINE="${custom_schedule} ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
;;
|
|
5)
|
|
echo "Skipping cron setup. You can add manually with:"
|
|
echo " crontab -e"
|
|
echo " Add: 0 7 * * * ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
CRON_LINE=""
|
|
;;
|
|
*)
|
|
echo "Invalid choice. Skipping cron setup."
|
|
CRON_LINE=""
|
|
;;
|
|
esac
|
|
|
|
if [ -n "$CRON_LINE" ]; then
|
|
echo ""
|
|
echo "Adding to crontab: $CRON_LINE"
|
|
|
|
if [ "$EUID" -eq 0 ]; then
|
|
# Add as target user
|
|
(su - $TARGET_USER -c "crontab -l" 2>/dev/null || true; echo "$CRON_LINE") | \
|
|
su - $TARGET_USER -c "crontab -"
|
|
else
|
|
# Add as current user
|
|
(crontab -l 2>/dev/null || true; echo "$CRON_LINE") | crontab -
|
|
fi
|
|
|
|
echo "✅ Cron job added!"
|
|
echo ""
|
|
echo "View with: crontab -l"
|
|
fi
|
|
|
|
# Summary
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " Setup Complete!"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "📝 What was configured:"
|
|
echo " ✅ Scripts made executable"
|
|
echo " ✅ Logs directory created: ${POTE_DIR}/logs"
|
|
if [ -n "$CRON_LINE" ]; then
|
|
echo " ✅ Cron job scheduled"
|
|
fi
|
|
echo ""
|
|
echo "🧪 Test manually:"
|
|
echo " ${POTE_DIR}/scripts/daily_fetch.sh"
|
|
echo ""
|
|
echo "📊 View logs:"
|
|
echo " tail -f ${POTE_DIR}/logs/daily_fetch_\$(date +%Y%m%d).log"
|
|
echo ""
|
|
echo "⚙️ Manage cron:"
|
|
echo " crontab -l # View cron jobs"
|
|
echo " crontab -e # Edit cron jobs"
|
|
echo ""
|
|
echo "📚 Documentation:"
|
|
echo " ${POTE_DIR}/docs/10_automation.md"
|
|
echo ""
|
|
|