Initial commit: POTE Phase 1 complete
- PR1: Project scaffold, DB models, price loader - PR2: Congressional trade ingestion (House Stock Watcher) - PR3: Security enrichment + deployment infrastructure - 37 passing tests, 87%+ coverage - Docker + Proxmox deployment ready - Complete documentation - Works 100% offline with fixtures
This commit is contained in:
ilia
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enrich securities with data from yfinance (names, sectors, industries).
|
||||
Usage: python scripts/enrich_securities.py [--ticker TICKER] [--limit N] [--force]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from pote.db import SessionLocal
|
||||
from pote.ingestion.security_enricher import SecurityEnricher
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
"""Enrich securities with yfinance data."""
|
||||
parser = argparse.ArgumentParser(description="Enrich securities with yfinance data")
|
||||
parser.add_argument("--ticker", type=str, help="Enrich a specific ticker")
|
||||
parser.add_argument("--limit", type=int, help="Maximum number of securities to enrich")
|
||||
parser.add_argument(
|
||||
"--force", action="store_true", help="Re-enrich already enriched securities"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.info("=== Security Enrichment (yfinance) ===")
|
||||
|
||||
try:
|
||||
with SessionLocal() as session:
|
||||
enricher = SecurityEnricher(session)
|
||||
|
||||
if args.ticker:
|
||||
logger.info(f"Enriching single ticker: {args.ticker}")
|
||||
success = enricher.enrich_by_ticker(args.ticker)
|
||||
if success:
|
||||
logger.info(f"✓ Successfully enriched {args.ticker}")
|
||||
else:
|
||||
logger.error(f"✗ Failed to enrich {args.ticker}")
|
||||
return 1
|
||||
else:
|
||||
logger.info(f"Enriching {'all' if not args.limit else args.limit} securities")
|
||||
if args.force:
|
||||
logger.info("Force mode: re-enriching already enriched securities")
|
||||
|
||||
counts = enricher.enrich_all_securities(limit=args.limit, force=args.force)
|
||||
|
||||
logger.info("\n=== Summary ===")
|
||||
logger.info(f"Total processed: {counts['total']}")
|
||||
logger.info(f"✓ Successfully enriched: {counts['enriched']}")
|
||||
logger.info(f"✗ Failed: {counts['failed']}")
|
||||
|
||||
logger.info("\n✅ Done!")
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Enrichment failed: {e}", exc_info=True)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch recent congressional trades from House Stock Watcher and ingest into DB.
|
||||
Usage: python scripts/fetch_congressional_trades.py [--days N] [--limit N]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from pote.db import SessionLocal
|
||||
from pote.ingestion.house_watcher import HouseWatcherClient
|
||||
from pote.ingestion.trade_loader import TradeLoader
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
"""Fetch and ingest congressional trades."""
|
||||
parser = argparse.ArgumentParser(description="Fetch congressional trades (free API)")
|
||||
parser.add_argument(
|
||||
"--days", type=int, default=30, help="Number of days to look back (default: 30)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=None, help="Maximum number of transactions to fetch"
|
||||
)
|
||||
parser.add_argument("--all", action="store_true", help="Fetch all transactions (ignore --days)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.info("=== Fetching Congressional Trades from House Stock Watcher ===")
|
||||
logger.info("Source: https://housestockwatcher.com (free, no API key)")
|
||||
|
||||
try:
|
||||
with HouseWatcherClient() as client:
|
||||
if args.all:
|
||||
logger.info(f"Fetching all transactions (limit={args.limit})")
|
||||
transactions = client.fetch_all_transactions(limit=args.limit)
|
||||
else:
|
||||
logger.info(f"Fetching transactions from last {args.days} days")
|
||||
transactions = client.fetch_recent_transactions(days=args.days)
|
||||
|
||||
if args.limit:
|
||||
transactions = transactions[: args.limit]
|
||||
|
||||
if not transactions:
|
||||
logger.warning("No transactions fetched!")
|
||||
return
|
||||
|
||||
logger.info(f"Fetched {len(transactions)} transactions")
|
||||
|
||||
# Show sample
|
||||
logger.info("\nSample transaction:")
|
||||
sample = transactions[0]
|
||||
for key, val in sample.items():
|
||||
logger.info(f" {key}: {val}")
|
||||
|
||||
# Ingest into database
|
||||
logger.info("\n=== Ingesting into database ===")
|
||||
with SessionLocal() as session:
|
||||
loader = TradeLoader(session)
|
||||
counts = loader.ingest_transactions(transactions)
|
||||
|
||||
logger.info("\n=== Summary ===")
|
||||
logger.info(f"✓ Officials created/updated: {counts['officials']}")
|
||||
logger.info(f"✓ Securities created/updated: {counts['securities']}")
|
||||
logger.info(f"✓ Trades ingested: {counts['trades']}")
|
||||
|
||||
# Query some stats
|
||||
with SessionLocal() as session:
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from pote.db.models import Official, Trade
|
||||
|
||||
total_trades = session.scalar(select(func.count(Trade.id)))
|
||||
total_officials = session.scalar(select(func.count(Official.id)))
|
||||
|
||||
logger.info("\nDatabase totals:")
|
||||
logger.info(f" Total officials: {total_officials}")
|
||||
logger.info(f" Total trades: {total_trades}")
|
||||
|
||||
logger.info("\n✅ Done!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch/ingest trades: {e}", exc_info=True)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick smoke-test: fetch price data for a few tickers.
|
||||
Usage: python scripts/fetch_sample_prices.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
|
||||
from pote.db import SessionLocal
|
||||
from pote.ingestion.prices import PriceLoader
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
"""Fetch sample price data."""
|
||||
tickers = ["AAPL", "MSFT", "TSLA"]
|
||||
end_date = date.today()
|
||||
start_date = end_date - timedelta(days=30) # Last 30 days
|
||||
|
||||
with SessionLocal() as session:
|
||||
loader = PriceLoader(session)
|
||||
logger.info(f"Fetching prices for {tickers} from {start_date} to {end_date}")
|
||||
|
||||
results = loader.bulk_fetch_prices(tickers, start_date, end_date)
|
||||
|
||||
for ticker, count in results.items():
|
||||
logger.info(f" {ticker}: {count} records")
|
||||
|
||||
logger.info("Done!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Ingest sample congressional trades from fixture files (no network required).
|
||||
Usage: python scripts/ingest_from_fixtures.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from pote.db import SessionLocal
|
||||
from pote.ingestion.trade_loader import TradeLoader
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
"""Ingest sample trades from fixtures."""
|
||||
logger.info("=== Ingesting Sample Congressional Trades from Fixtures ===")
|
||||
logger.info("(No network required - using test fixtures)")
|
||||
|
||||
# Load fixture
|
||||
fixture_path = Path(__file__).parent.parent / "tests" / "fixtures" / "sample_house_watcher.json"
|
||||
|
||||
if not fixture_path.exists():
|
||||
logger.error(f"Fixture file not found: {fixture_path}")
|
||||
return 1
|
||||
|
||||
with open(fixture_path) as f:
|
||||
transactions = json.load(f)
|
||||
|
||||
logger.info(f"Loaded {len(transactions)} sample transactions from fixture")
|
||||
|
||||
# Show sample
|
||||
logger.info("\nSample transaction:")
|
||||
sample = transactions[0]
|
||||
for key, val in sample.items():
|
||||
logger.info(f" {key}: {val}")
|
||||
|
||||
# Ingest into database
|
||||
logger.info("\n=== Ingesting into database ===")
|
||||
with SessionLocal() as session:
|
||||
loader = TradeLoader(session)
|
||||
counts = loader.ingest_transactions(transactions)
|
||||
|
||||
logger.info("\n=== Summary ===")
|
||||
logger.info(f"✓ Officials created/updated: {counts['officials']}")
|
||||
logger.info(f"✓ Securities created/updated: {counts['securities']}")
|
||||
logger.info(f"✓ Trades ingested: {counts['trades']}")
|
||||
|
||||
# Query some stats
|
||||
with SessionLocal() as session:
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from pote.db.models import Official, Trade
|
||||
|
||||
total_trades = session.scalar(select(func.count(Trade.id)))
|
||||
total_officials = session.scalar(select(func.count(Official.id)))
|
||||
|
||||
logger.info("\nDatabase totals:")
|
||||
logger.info(f" Total officials: {total_officials}")
|
||||
logger.info(f" Total trades: {total_trades}")
|
||||
|
||||
# Show some actual data
|
||||
logger.info("\n=== Sample Officials ===")
|
||||
with SessionLocal() as session:
|
||||
stmt = select(Official).limit(5)
|
||||
officials = session.scalars(stmt).all()
|
||||
for official in officials:
|
||||
stmt = select(func.count(Trade.id)).where(Trade.official_id == official.id)
|
||||
trade_count = session.scalar(stmt)
|
||||
logger.info(
|
||||
f" {official.name} ({official.chamber}, {official.party}): {trade_count} trades"
|
||||
)
|
||||
|
||||
logger.info("\n✅ Done! All sample data ingested successfully.")
|
||||
logger.info("Note: This works 100% offline using fixture files.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/bin/bash
|
||||
# POTE Proxmox/Ubuntu Setup Script
|
||||
# Run this inside your Proxmox LXC container or Ubuntu VM
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo " POTE - Proxmox Deployment Setup"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
POTE_USER="poteapp"
|
||||
POTE_HOME="/home/$POTE_USER"
|
||||
POTE_DIR="$POTE_HOME/pote"
|
||||
DB_NAME="pote"
|
||||
DB_USER="poteuser"
|
||||
DB_PASS="changeme123" # CHANGE THIS!
|
||||
|
||||
echo -e "${YELLOW}⚠️ Using default password '$DB_PASS' - CHANGE THIS in production!${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if running as root
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root (sudo)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 1: Update system
|
||||
echo -e "${GREEN}[1/9]${NC} Updating system..."
|
||||
apt update && apt upgrade -y
|
||||
|
||||
# Step 2: Install dependencies
|
||||
echo -e "${GREEN}[2/9]${NC} Installing dependencies..."
|
||||
apt install -y \
|
||||
python3.11 \
|
||||
python3.11-venv \
|
||||
python3-pip \
|
||||
postgresql \
|
||||
postgresql-contrib \
|
||||
git \
|
||||
curl \
|
||||
build-essential \
|
||||
libpq-dev \
|
||||
nano \
|
||||
htop
|
||||
|
||||
# Step 3: Setup PostgreSQL
|
||||
echo -e "${GREEN}[3/9]${NC} Setting up PostgreSQL..."
|
||||
sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || \
|
||||
sudo -u postgres psql << EOF
|
||||
CREATE DATABASE $DB_NAME;
|
||||
CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';
|
||||
GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;
|
||||
ALTER DATABASE $DB_NAME OWNER TO $DB_USER;
|
||||
EOF
|
||||
|
||||
echo "✓ PostgreSQL database '$DB_NAME' created"
|
||||
|
||||
# Step 4: Create app user
|
||||
echo -e "${GREEN}[4/9]${NC} Creating application user..."
|
||||
id -u $POTE_USER &>/dev/null || useradd -m -s /bin/bash $POTE_USER
|
||||
echo "✓ User '$POTE_USER' created"
|
||||
|
||||
# Step 5: Clone repository (if not exists)
|
||||
echo -e "${GREEN}[5/9]${NC} Setting up POTE repository..."
|
||||
if [ ! -d "$POTE_DIR" ]; then
|
||||
echo "Enter your POTE repository URL (or press Enter to skip git clone):"
|
||||
read -r REPO_URL
|
||||
|
||||
if [ -n "$REPO_URL" ]; then
|
||||
sudo -u $POTE_USER git clone "$REPO_URL" "$POTE_DIR"
|
||||
else
|
||||
echo "Skipping git clone. Make sure code is in $POTE_DIR"
|
||||
fi
|
||||
else
|
||||
echo "✓ Directory $POTE_DIR already exists"
|
||||
fi
|
||||
|
||||
# Step 6: Setup Python environment
|
||||
echo -e "${GREEN}[6/9]${NC} Setting up Python environment..."
|
||||
sudo -u $POTE_USER bash << 'EOF'
|
||||
cd $POTE_DIR
|
||||
python3.11 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install --upgrade pip
|
||||
pip install -e .
|
||||
echo "✓ Python dependencies installed"
|
||||
EOF
|
||||
|
||||
# Step 7: Create .env file
|
||||
echo -e "${GREEN}[7/9]${NC} Creating environment configuration..."
|
||||
sudo -u $POTE_USER bash << EOF
|
||||
cat > $POTE_DIR/.env << ENVEOF
|
||||
DATABASE_URL=postgresql://$DB_USER:$DB_PASS@localhost:5432/$DB_NAME
|
||||
QUIVERQUANT_API_KEY=
|
||||
FMP_API_KEY=
|
||||
LOG_LEVEL=INFO
|
||||
ENVEOF
|
||||
chmod 600 $POTE_DIR/.env
|
||||
EOF
|
||||
echo "✓ Environment file created"
|
||||
|
||||
# Step 8: Run database migrations
|
||||
echo -e "${GREEN}[8/9]${NC} Running database migrations..."
|
||||
sudo -u $POTE_USER bash << 'EOF'
|
||||
cd $POTE_DIR
|
||||
source venv/bin/activate
|
||||
alembic upgrade head
|
||||
EOF
|
||||
echo "✓ Database schema initialized"
|
||||
|
||||
# Step 9: Setup directories
|
||||
echo -e "${GREEN}[9/9]${NC} Creating directories..."
|
||||
sudo -u $POTE_USER mkdir -p $POTE_HOME/logs
|
||||
sudo -u $POTE_USER mkdir -p $POTE_HOME/backups
|
||||
echo "✓ Log and backup directories created"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo " ✅ POTE Installation Complete!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo ""
|
||||
echo "1. Switch to pote user:"
|
||||
echo " su - $POTE_USER"
|
||||
echo ""
|
||||
echo "2. Activate virtual environment:"
|
||||
echo " cd pote && source venv/bin/activate"
|
||||
echo ""
|
||||
echo "3. Test with fixtures (offline):"
|
||||
echo " python scripts/ingest_from_fixtures.py"
|
||||
echo ""
|
||||
echo "4. Enrich securities:"
|
||||
echo " python scripts/enrich_securities.py"
|
||||
echo ""
|
||||
echo "5. Setup cron jobs (as poteapp user):"
|
||||
echo " crontab -e"
|
||||
echo ""
|
||||
echo " Add these lines:"
|
||||
echo " 0 6 * * * cd $POTE_DIR && $POTE_DIR/venv/bin/python scripts/fetch_congressional_trades.py --days 7 >> $POTE_HOME/logs/trades.log 2>&1"
|
||||
echo " 15 6 * * * cd $POTE_DIR && $POTE_DIR/venv/bin/python scripts/enrich_securities.py >> $POTE_HOME/logs/enrich.log 2>&1"
|
||||
echo ""
|
||||
echo "⚠️ IMPORTANT: Change database password in .env!"
|
||||
echo " Edit: $POTE_DIR/.env"
|
||||
echo ""
|
||||
echo "📖 Full guide: docs/08_proxmox_deployment.md"
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user