Initial commit: POTE Phase 1 complete
- PR1: Project scaffold, DB models, price loader - PR2: Congressional trade ingestion (House Stock Watcher) - PR3: Security enrichment + deployment infrastructure - 37 passing tests, 87%+ coverage - Docker + Proxmox deployment ready - Complete documentation - Works 100% offline with fixtures
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Tests for POTE."""
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Pytest fixtures and test configuration.
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from pote.db import Base
|
||||
from pote.db.models import Official, Price, Security, Trade
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def test_db_session() -> Session:
|
||||
"""
|
||||
Create an in-memory SQLite database for testing.
|
||||
Each test gets a fresh database.
|
||||
"""
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
TestSessionLocal = sessionmaker(bind=engine)
|
||||
session = TestSessionLocal()
|
||||
|
||||
yield session
|
||||
|
||||
session.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_official(test_db_session: Session) -> Official:
|
||||
"""Create a sample official for testing."""
|
||||
official = Official(
|
||||
name="Jane Doe",
|
||||
chamber="Senate",
|
||||
party="Independent",
|
||||
state="CA",
|
||||
bioguide_id="D000123",
|
||||
)
|
||||
test_db_session.add(official)
|
||||
test_db_session.commit()
|
||||
test_db_session.refresh(official)
|
||||
return official
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_security(test_db_session: Session) -> Security:
|
||||
"""Create a sample security for testing."""
|
||||
security = Security(
|
||||
ticker="AAPL",
|
||||
name="Apple Inc.",
|
||||
exchange="NASDAQ",
|
||||
sector="Technology",
|
||||
asset_type="stock",
|
||||
)
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
test_db_session.refresh(security)
|
||||
return security
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_trade(
|
||||
test_db_session: Session, sample_official: Official, sample_security: Security
|
||||
) -> Trade:
|
||||
"""Create a sample trade for testing."""
|
||||
trade = Trade(
|
||||
official_id=sample_official.id,
|
||||
security_id=sample_security.id,
|
||||
source="test",
|
||||
external_id="test-001",
|
||||
transaction_date=date(2024, 1, 15),
|
||||
filing_date=date(2024, 2, 1),
|
||||
side="buy",
|
||||
value_min=Decimal("15000.00"),
|
||||
value_max=Decimal("50000.00"),
|
||||
currency="USD",
|
||||
)
|
||||
test_db_session.add(trade)
|
||||
test_db_session.commit()
|
||||
test_db_session.refresh(trade)
|
||||
return trade
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_price(test_db_session: Session, sample_security: Security) -> Price:
|
||||
"""Create a sample price record for testing."""
|
||||
price = Price(
|
||||
security_id=sample_security.id,
|
||||
date=date(2024, 1, 15),
|
||||
open=Decimal("180.50"),
|
||||
high=Decimal("182.75"),
|
||||
low=Decimal("179.00"),
|
||||
close=Decimal("181.25"),
|
||||
volume=50000000,
|
||||
source="yfinance",
|
||||
)
|
||||
test_db_session.add(price)
|
||||
test_db_session.commit()
|
||||
test_db_session.refresh(price)
|
||||
return price
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
[
|
||||
{
|
||||
"representative": "Nancy Pelosi",
|
||||
"ticker": "NVDA",
|
||||
"transaction_date": "01/15/2024",
|
||||
"disclosure_date": "02/01/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"district": "CA-11",
|
||||
"party": "Democrat",
|
||||
"cap_gains_over_200_usd": false
|
||||
},
|
||||
{
|
||||
"representative": "Josh Gottheimer",
|
||||
"ticker": "MSFT",
|
||||
"transaction_date": "01/20/2024",
|
||||
"disclosure_date": "02/05/2024",
|
||||
"transaction": "Sale",
|
||||
"amount": "$15,001 - $50,000",
|
||||
"house": "House",
|
||||
"district": "NJ-05",
|
||||
"party": "Democrat",
|
||||
"cap_gains_over_200_usd": false
|
||||
},
|
||||
{
|
||||
"representative": "Tommy Tuberville",
|
||||
"ticker": "AAPL",
|
||||
"transaction_date": "01/10/2024",
|
||||
"disclosure_date": "01/30/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$50,001 - $100,000",
|
||||
"house": "Senate",
|
||||
"district": "",
|
||||
"party": "Republican",
|
||||
"cap_gains_over_200_usd": false
|
||||
},
|
||||
{
|
||||
"representative": "Dan Crenshaw",
|
||||
"ticker": "TSLA",
|
||||
"transaction_date": "01/18/2024",
|
||||
"disclosure_date": "02/03/2024",
|
||||
"transaction": "Sale",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"district": "TX-02",
|
||||
"party": "Republican",
|
||||
"cap_gains_over_200_usd": true
|
||||
},
|
||||
{
|
||||
"representative": "Nancy Pelosi",
|
||||
"ticker": "GOOGL",
|
||||
"transaction_date": "01/22/2024",
|
||||
"disclosure_date": "02/10/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$15,001 - $50,000",
|
||||
"house": "House",
|
||||
"district": "CA-11",
|
||||
"party": "Democrat",
|
||||
"cap_gains_over_200_usd": false
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Tests for House Stock Watcher client.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from pote.ingestion.house_watcher import (
|
||||
HouseWatcherClient,
|
||||
normalize_transaction_type,
|
||||
parse_amount_range,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_amount_range_with_range():
|
||||
"""Test parsing amount range string."""
|
||||
min_val, max_val = parse_amount_range("$1,001 - $15,000")
|
||||
assert min_val == 1001.0
|
||||
assert max_val == 15000.0
|
||||
|
||||
|
||||
def test_parse_amount_range_single_value():
|
||||
"""Test parsing single value."""
|
||||
min_val, max_val = parse_amount_range("$25,000")
|
||||
assert min_val == 25000.0
|
||||
assert max_val == 25000.0
|
||||
|
||||
|
||||
def test_parse_amount_range_invalid():
|
||||
"""Test parsing invalid amount."""
|
||||
min_val, max_val = parse_amount_range("N/A")
|
||||
assert min_val is None
|
||||
assert max_val is None
|
||||
|
||||
|
||||
def test_normalize_transaction_type():
|
||||
"""Test normalizing transaction types."""
|
||||
assert normalize_transaction_type("Purchase") == "buy"
|
||||
assert normalize_transaction_type("Sale") == "sell"
|
||||
assert normalize_transaction_type("Exchange") == "exchange"
|
||||
assert normalize_transaction_type("purchase") == "buy"
|
||||
assert normalize_transaction_type("SALE") == "sell"
|
||||
|
||||
|
||||
@patch("pote.ingestion.house_watcher.httpx.Client")
|
||||
def test_fetch_all_transactions(mock_client_class):
|
||||
"""Test fetching all transactions."""
|
||||
# Mock response
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = [
|
||||
{
|
||||
"representative": "Test Official",
|
||||
"ticker": "AAPL",
|
||||
"transaction_date": "2024-01-15",
|
||||
"disclosure_date": "2024-02-01",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"party": "Independent",
|
||||
}
|
||||
]
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_client_instance.get.return_value = mock_response
|
||||
mock_client_class.return_value = mock_client_instance
|
||||
|
||||
with HouseWatcherClient() as client:
|
||||
txns = client.fetch_all_transactions()
|
||||
|
||||
assert len(txns) == 1
|
||||
assert txns[0]["ticker"] == "AAPL"
|
||||
assert txns[0]["representative"] == "Test Official"
|
||||
|
||||
|
||||
@patch("pote.ingestion.house_watcher.httpx.Client")
|
||||
def test_fetch_all_transactions_with_limit(mock_client_class):
|
||||
"""Test fetching transactions with limit."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = [{"id": i} for i in range(100)]
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_client_instance.get.return_value = mock_response
|
||||
mock_client_class.return_value = mock_client_instance
|
||||
|
||||
with HouseWatcherClient() as client:
|
||||
txns = client.fetch_all_transactions(limit=10)
|
||||
|
||||
assert len(txns) == 10
|
||||
|
||||
|
||||
@patch("pote.ingestion.house_watcher.httpx.Client")
|
||||
def test_fetch_recent_transactions(mock_client_class):
|
||||
"""Test filtering to recent transactions."""
|
||||
from datetime import date, timedelta
|
||||
|
||||
today = date.today()
|
||||
recent_date = (today - timedelta(days=5)).strftime("%m/%d/%Y")
|
||||
old_date = (today - timedelta(days=100)).strftime("%m/%d/%Y")
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = [
|
||||
{"disclosure_date": recent_date, "ticker": "AAPL"},
|
||||
{"disclosure_date": old_date, "ticker": "MSFT"},
|
||||
{"disclosure_date": recent_date, "ticker": "GOOGL"},
|
||||
]
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_client_instance.get.return_value = mock_response
|
||||
mock_client_class.return_value = mock_client_instance
|
||||
|
||||
with HouseWatcherClient() as client:
|
||||
recent = client.fetch_recent_transactions(days=30)
|
||||
|
||||
assert len(recent) == 2
|
||||
assert recent[0]["ticker"] == "AAPL"
|
||||
assert recent[1]["ticker"] == "GOOGL"
|
||||
|
||||
|
||||
def test_house_watcher_client_context_manager():
|
||||
"""Test client as context manager."""
|
||||
with HouseWatcherClient() as client:
|
||||
assert client is not None
|
||||
# Verify close was called (client should be closed after context)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
Tests for database models.
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from pote.db.models import Price, Security, Trade
|
||||
|
||||
|
||||
def test_create_official(test_db_session, sample_official):
|
||||
"""Test creating an official."""
|
||||
assert sample_official.id is not None
|
||||
assert sample_official.name == "Jane Doe"
|
||||
assert sample_official.chamber == "Senate"
|
||||
assert sample_official.party == "Independent"
|
||||
assert sample_official.state == "CA"
|
||||
|
||||
|
||||
def test_create_security(test_db_session, sample_security):
|
||||
"""Test creating a security."""
|
||||
assert sample_security.id is not None
|
||||
assert sample_security.ticker == "AAPL"
|
||||
assert sample_security.name == "Apple Inc."
|
||||
assert sample_security.sector == "Technology"
|
||||
|
||||
|
||||
def test_create_trade(test_db_session, sample_trade):
|
||||
"""Test creating a trade with relationships."""
|
||||
assert sample_trade.id is not None
|
||||
assert sample_trade.official_id is not None
|
||||
assert sample_trade.security_id is not None
|
||||
assert sample_trade.side == "buy"
|
||||
assert sample_trade.value_min == Decimal("15000.00")
|
||||
|
||||
# Test relationships
|
||||
assert sample_trade.official.name == "Jane Doe"
|
||||
assert sample_trade.security.ticker == "AAPL"
|
||||
|
||||
|
||||
def test_create_price(test_db_session, sample_price):
|
||||
"""Test creating a price record."""
|
||||
assert sample_price.id is not None
|
||||
assert sample_price.close == Decimal("181.25")
|
||||
assert sample_price.volume == 50000000
|
||||
assert sample_price.security.ticker == "AAPL"
|
||||
|
||||
|
||||
def test_unique_constraints(test_db_session, sample_security):
|
||||
"""Test that unique constraints work."""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
# Try to create duplicate security with same ticker
|
||||
dup_security = Security(ticker="AAPL", name="Apple Duplicate")
|
||||
test_db_session.add(dup_security)
|
||||
|
||||
try:
|
||||
test_db_session.commit()
|
||||
assert False, "Should have raised IntegrityError"
|
||||
except IntegrityError:
|
||||
test_db_session.rollback()
|
||||
# Expected behavior
|
||||
|
||||
|
||||
def test_price_unique_per_security_date(test_db_session, sample_security):
|
||||
"""Test that we can't have duplicate prices for same security/date."""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
price1 = Price(
|
||||
security_id=sample_security.id,
|
||||
date=date(2024, 1, 1),
|
||||
close=Decimal("100.00"),
|
||||
)
|
||||
test_db_session.add(price1)
|
||||
test_db_session.commit()
|
||||
|
||||
price2 = Price(
|
||||
security_id=sample_security.id,
|
||||
date=date(2024, 1, 1),
|
||||
close=Decimal("101.00"),
|
||||
)
|
||||
test_db_session.add(price2)
|
||||
|
||||
try:
|
||||
test_db_session.commit()
|
||||
assert False, "Should have raised IntegrityError"
|
||||
except IntegrityError:
|
||||
test_db_session.rollback()
|
||||
# Expected behavior
|
||||
|
||||
|
||||
def test_trade_queries(test_db_session, sample_official, sample_security):
|
||||
"""Test querying trades by official and date range."""
|
||||
# Create multiple trades
|
||||
trades_data = [
|
||||
{"date": date(2024, 1, 10), "side": "buy"},
|
||||
{"date": date(2024, 1, 15), "side": "sell"},
|
||||
{"date": date(2024, 2, 1), "side": "buy"},
|
||||
]
|
||||
|
||||
for i, td in enumerate(trades_data):
|
||||
trade = Trade(
|
||||
official_id=sample_official.id,
|
||||
security_id=sample_security.id,
|
||||
source="test",
|
||||
external_id=f"test-{i}",
|
||||
transaction_date=td["date"],
|
||||
side=td["side"],
|
||||
value_min=Decimal("10000.00"),
|
||||
value_max=Decimal("50000.00"),
|
||||
)
|
||||
test_db_session.add(trade)
|
||||
test_db_session.commit()
|
||||
|
||||
# Query trades in January
|
||||
stmt = (
|
||||
select(Trade)
|
||||
.where(Trade.official_id == sample_official.id)
|
||||
.where(Trade.transaction_date >= date(2024, 1, 1))
|
||||
.where(Trade.transaction_date < date(2024, 2, 1))
|
||||
.order_by(Trade.transaction_date)
|
||||
)
|
||||
jan_trades = test_db_session.scalars(stmt).all()
|
||||
|
||||
assert len(jan_trades) == 2
|
||||
assert jan_trades[0].transaction_date == date(2024, 1, 10)
|
||||
assert jan_trades[1].transaction_date == date(2024, 1, 15)
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
Tests for price loader.
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from pote.db.models import Price, Security
|
||||
from pote.ingestion.prices import PriceLoader
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def price_loader(test_db_session):
|
||||
"""Create a PriceLoader instance with test session."""
|
||||
return PriceLoader(test_db_session)
|
||||
|
||||
|
||||
def test_get_or_create_security_new(price_loader, test_db_session):
|
||||
"""Test creating a new security."""
|
||||
security = price_loader._get_or_create_security("MSFT")
|
||||
|
||||
assert security.id is not None
|
||||
assert security.ticker == "MSFT"
|
||||
assert security.asset_type == "stock"
|
||||
|
||||
# Verify it's in the database
|
||||
stmt = select(Security).where(Security.ticker == "MSFT")
|
||||
db_security = test_db_session.scalars(stmt).first()
|
||||
assert db_security is not None
|
||||
assert db_security.id == security.id
|
||||
|
||||
|
||||
def test_get_or_create_security_existing(price_loader, test_db_session, sample_security):
|
||||
"""Test getting an existing security."""
|
||||
security = price_loader._get_or_create_security("AAPL")
|
||||
|
||||
assert security.id == sample_security.id
|
||||
assert security.ticker == "AAPL"
|
||||
|
||||
# Verify no duplicate was created
|
||||
stmt = select(Security).where(Security.ticker == "AAPL")
|
||||
count = len(test_db_session.scalars(stmt).all())
|
||||
assert count == 1
|
||||
|
||||
|
||||
def test_store_prices(price_loader, test_db_session, sample_security):
|
||||
"""Test storing price data."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"date": [date(2024, 1, 1), date(2024, 1, 2), date(2024, 1, 3)],
|
||||
"open": [100.0, 101.0, 102.0],
|
||||
"high": [105.0, 106.0, 107.0],
|
||||
"low": [99.0, 100.0, 101.0],
|
||||
"close": [103.0, 104.0, 105.0],
|
||||
"volume": [1000000, 1100000, 1200000],
|
||||
}
|
||||
)
|
||||
|
||||
count = price_loader._store_prices(sample_security.id, df)
|
||||
|
||||
assert count == 3
|
||||
|
||||
# Verify prices in database
|
||||
stmt = select(Price).where(Price.security_id == sample_security.id).order_by(Price.date)
|
||||
prices = test_db_session.scalars(stmt).all()
|
||||
|
||||
assert len(prices) == 3
|
||||
assert prices[0].date == date(2024, 1, 1)
|
||||
assert prices[0].close == Decimal("103.0")
|
||||
assert prices[2].volume == 1200000
|
||||
|
||||
|
||||
def test_store_prices_upsert(price_loader, test_db_session, sample_security):
|
||||
"""Test that storing prices twice performs upsert (update on conflict)."""
|
||||
df1 = pd.DataFrame(
|
||||
{
|
||||
"date": [date(2024, 1, 1)],
|
||||
"open": [100.0],
|
||||
"high": [105.0],
|
||||
"low": [99.0],
|
||||
"close": [103.0],
|
||||
"volume": [1000000],
|
||||
}
|
||||
)
|
||||
|
||||
count1 = price_loader._store_prices(sample_security.id, df1)
|
||||
assert count1 == 1
|
||||
|
||||
# Store again with updated values
|
||||
df2 = pd.DataFrame(
|
||||
{
|
||||
"date": [date(2024, 1, 1)],
|
||||
"open": [100.5],
|
||||
"high": [106.0],
|
||||
"low": [99.5],
|
||||
"close": [104.0],
|
||||
"volume": [1100000],
|
||||
}
|
||||
)
|
||||
|
||||
count2 = price_loader._store_prices(sample_security.id, df2)
|
||||
assert count2 == 1
|
||||
|
||||
# Verify only one price exists with updated values
|
||||
stmt = select(Price).where(Price.security_id == sample_security.id)
|
||||
prices = test_db_session.scalars(stmt).all()
|
||||
|
||||
assert len(prices) == 1
|
||||
assert prices[0].close == Decimal("104.0")
|
||||
assert prices[0].volume == 1100000
|
||||
|
||||
|
||||
def test_get_missing_date_range_start_no_data(price_loader, test_db_session, sample_security):
|
||||
"""Test finding missing date range when no data exists."""
|
||||
start = date(2024, 1, 1)
|
||||
end = date(2024, 1, 31)
|
||||
|
||||
missing_start = price_loader._get_missing_date_range_start(sample_security.id, start, end)
|
||||
|
||||
assert missing_start == start
|
||||
|
||||
|
||||
def test_get_missing_date_range_start_partial_data(price_loader, test_db_session, sample_security):
|
||||
"""Test finding missing date range when partial data exists."""
|
||||
# Add prices for first week of January
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"date": [date(2024, 1, d) for d in range(1, 8)],
|
||||
"close": [100.0 + d for d in range(7)],
|
||||
}
|
||||
)
|
||||
price_loader._store_prices(sample_security.id, df)
|
||||
|
||||
start = date(2024, 1, 1)
|
||||
end = date(2024, 1, 31)
|
||||
|
||||
missing_start = price_loader._get_missing_date_range_start(sample_security.id, start, end)
|
||||
|
||||
# Should start from day after last existing (Jan 8)
|
||||
assert missing_start == date(2024, 1, 8)
|
||||
|
||||
|
||||
@patch("pote.ingestion.prices.yf.Ticker")
|
||||
def test_fetch_and_store_prices_integration(mock_ticker, price_loader, test_db_session):
|
||||
"""Test the full fetch_and_store_prices flow with mocked yfinance."""
|
||||
# Mock yfinance response
|
||||
mock_hist_df = pd.DataFrame(
|
||||
{
|
||||
"Date": pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-03"]),
|
||||
"Open": [100.0, 101.0, 102.0],
|
||||
"High": [105.0, 106.0, 107.0],
|
||||
"Low": [99.0, 100.0, 101.0],
|
||||
"Close": [103.0, 104.0, 105.0],
|
||||
"Volume": [1000000, 1100000, 1200000],
|
||||
}
|
||||
).set_index("Date")
|
||||
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.history.return_value = mock_hist_df
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
# Fetch and store
|
||||
count = price_loader.fetch_and_store_prices(
|
||||
"TSLA",
|
||||
start_date=date(2024, 1, 1),
|
||||
end_date=date(2024, 1, 3),
|
||||
)
|
||||
|
||||
assert count == 3
|
||||
|
||||
# Verify security was created
|
||||
stmt = select(Security).where(Security.ticker == "TSLA")
|
||||
security = test_db_session.scalars(stmt).first()
|
||||
assert security is not None
|
||||
|
||||
# Verify prices were stored
|
||||
stmt = select(Price).where(Price.security_id == security.id).order_by(Price.date)
|
||||
prices = test_db_session.scalars(stmt).all()
|
||||
|
||||
assert len(prices) == 3
|
||||
assert prices[0].close == Decimal("103.0")
|
||||
assert prices[2].close == Decimal("105.0")
|
||||
|
||||
|
||||
@patch("pote.ingestion.prices.yf.Ticker")
|
||||
def test_fetch_and_store_prices_idempotent(mock_ticker, price_loader, test_db_session):
|
||||
"""Test that re-fetching doesn't duplicate data."""
|
||||
mock_hist_df = pd.DataFrame(
|
||||
{
|
||||
"Date": pd.to_datetime(["2024-01-01"]),
|
||||
"Open": [100.0],
|
||||
"High": [105.0],
|
||||
"Low": [99.0],
|
||||
"Close": [103.0],
|
||||
"Volume": [1000000],
|
||||
}
|
||||
).set_index("Date")
|
||||
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.history.return_value = mock_hist_df
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
# Fetch twice
|
||||
count1 = price_loader.fetch_and_store_prices("TSLA", date(2024, 1, 1), date(2024, 1, 1))
|
||||
count2 = price_loader.fetch_and_store_prices("TSLA", date(2024, 1, 1), date(2024, 1, 1))
|
||||
|
||||
# First call should insert, second should skip (no missing dates)
|
||||
assert count1 == 1
|
||||
assert count2 == 0 # No missing data
|
||||
|
||||
# Verify only one price record exists
|
||||
stmt = select(Security).where(Security.ticker == "TSLA")
|
||||
security = test_db_session.scalars(stmt).first()
|
||||
|
||||
stmt = select(Price).where(Price.security_id == security.id)
|
||||
prices = test_db_session.scalars(stmt).all()
|
||||
assert len(prices) == 1
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
Tests for security enricher.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from pote.db.models import Security
|
||||
from pote.ingestion.security_enricher import SecurityEnricher
|
||||
|
||||
|
||||
def test_enrich_security_success(test_db_session):
|
||||
"""Test successful security enrichment."""
|
||||
# Create an unenriched security (name == ticker)
|
||||
security = Security(ticker="TSLA", name="TSLA", asset_type="stock")
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
# Mock yfinance response
|
||||
mock_info = {
|
||||
"symbol": "TSLA",
|
||||
"longName": "Tesla, Inc.",
|
||||
"sector": "Consumer Cyclical",
|
||||
"industry": "Auto Manufacturers",
|
||||
"exchange": "NASDAQ",
|
||||
"quoteType": "EQUITY",
|
||||
}
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = mock_info
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
success = enricher.enrich_security(security)
|
||||
|
||||
assert success is True
|
||||
assert security.name == "Tesla, Inc."
|
||||
assert security.sector == "Consumer Cyclical"
|
||||
assert security.industry == "Auto Manufacturers"
|
||||
assert security.exchange == "NASDAQ"
|
||||
assert security.asset_type == "stock"
|
||||
|
||||
|
||||
def test_enrich_security_etf(test_db_session):
|
||||
"""Test enriching an ETF."""
|
||||
security = Security(ticker="SPY", name="SPY", asset_type="stock")
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
mock_info = {
|
||||
"symbol": "SPY",
|
||||
"longName": "SPDR S&P 500 ETF Trust",
|
||||
"sector": None,
|
||||
"industry": None,
|
||||
"exchange": "NYSE",
|
||||
"quoteType": "ETF",
|
||||
}
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = mock_info
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
success = enricher.enrich_security(security)
|
||||
|
||||
assert success is True
|
||||
assert security.name == "SPDR S&P 500 ETF Trust"
|
||||
assert security.asset_type == "etf"
|
||||
|
||||
|
||||
def test_enrich_security_skip_already_enriched(test_db_session):
|
||||
"""Test that already enriched securities are skipped by default."""
|
||||
security = Security(
|
||||
ticker="MSFT",
|
||||
name="Microsoft Corporation", # Already enriched
|
||||
sector="Technology",
|
||||
asset_type="stock",
|
||||
)
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
# Should skip without calling yfinance
|
||||
success = enricher.enrich_security(security, force=False)
|
||||
assert success is False
|
||||
|
||||
|
||||
def test_enrich_security_force_refresh(test_db_session):
|
||||
"""Test force re-enrichment."""
|
||||
security = Security(
|
||||
ticker="GOOGL",
|
||||
name="Alphabet Inc.", # Already enriched
|
||||
sector="Technology",
|
||||
asset_type="stock",
|
||||
)
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
mock_info = {
|
||||
"symbol": "GOOGL",
|
||||
"longName": "Alphabet Inc. Class A", # Updated name
|
||||
"sector": "Communication Services", # Updated sector
|
||||
"industry": "Internet Content & Information",
|
||||
"exchange": "NASDAQ",
|
||||
"quoteType": "EQUITY",
|
||||
}
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = mock_info
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
success = enricher.enrich_security(security, force=True)
|
||||
|
||||
assert success is True
|
||||
assert security.name == "Alphabet Inc. Class A"
|
||||
assert security.sector == "Communication Services"
|
||||
|
||||
|
||||
def test_enrich_security_no_data(test_db_session, sample_security):
|
||||
"""Test handling of ticker with no data."""
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
# Mock empty response
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = {} # No data
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
success = enricher.enrich_security(sample_security)
|
||||
|
||||
assert success is False
|
||||
# Original values should be unchanged
|
||||
assert sample_security.name == "Apple Inc."
|
||||
|
||||
|
||||
def test_enrich_all_securities(test_db_session):
|
||||
"""Test enriching multiple securities."""
|
||||
# Create unenriched securities (name == ticker)
|
||||
securities = [
|
||||
Security(ticker="AAPL", name="AAPL", asset_type="stock"),
|
||||
Security(ticker="MSFT", name="MSFT", asset_type="stock"),
|
||||
Security(ticker="GOOGL", name="GOOGL", asset_type="stock"),
|
||||
]
|
||||
for sec in securities:
|
||||
test_db_session.add(sec)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
def mock_info_fn(ticker):
|
||||
return {
|
||||
"symbol": ticker,
|
||||
"longName": f"{ticker} Corporation",
|
||||
"sector": "Technology",
|
||||
"industry": "Software",
|
||||
"exchange": "NASDAQ",
|
||||
"quoteType": "EQUITY",
|
||||
}
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
|
||||
def side_effect(ticker_str):
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.info = mock_info_fn(ticker_str)
|
||||
return mock_instance
|
||||
|
||||
mock_ticker.side_effect = side_effect
|
||||
|
||||
counts = enricher.enrich_all_securities()
|
||||
|
||||
assert counts["total"] == 3
|
||||
assert counts["enriched"] == 3
|
||||
assert counts["failed"] == 0
|
||||
|
||||
# Verify enrichment
|
||||
stmt = select(Security).where(Security.ticker == "AAPL")
|
||||
aapl = test_db_session.scalars(stmt).first()
|
||||
assert aapl.name == "AAPL Corporation"
|
||||
assert aapl.sector == "Technology"
|
||||
|
||||
|
||||
def test_enrich_all_securities_with_limit(test_db_session):
|
||||
"""Test enriching with a limit."""
|
||||
# Create 5 unenriched securities
|
||||
for i in range(5):
|
||||
security = Security(ticker=f"TEST{i}", name=f"TEST{i}", asset_type="stock")
|
||||
test_db_session.add(security)
|
||||
test_db_session.commit()
|
||||
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = {
|
||||
"symbol": "TEST",
|
||||
"longName": "Test Corp",
|
||||
"quoteType": "EQUITY",
|
||||
}
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
counts = enricher.enrich_all_securities(limit=2)
|
||||
|
||||
assert counts["total"] == 2
|
||||
assert counts["enriched"] == 2
|
||||
|
||||
|
||||
def test_enrich_by_ticker_success(test_db_session, sample_security):
|
||||
"""Test enriching by specific ticker."""
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
mock_info = {
|
||||
"symbol": "AAPL",
|
||||
"longName": "Apple Inc.",
|
||||
"sector": "Technology",
|
||||
"quoteType": "EQUITY",
|
||||
}
|
||||
|
||||
with patch("pote.ingestion.security_enricher.yf.Ticker") as mock_ticker:
|
||||
mock_ticker_instance = MagicMock()
|
||||
mock_ticker_instance.info = mock_info
|
||||
mock_ticker.return_value = mock_ticker_instance
|
||||
|
||||
success = enricher.enrich_by_ticker("AAPL")
|
||||
|
||||
assert success is True
|
||||
|
||||
|
||||
def test_enrich_by_ticker_not_found(test_db_session):
|
||||
"""Test enriching a ticker not in database."""
|
||||
enricher = SecurityEnricher(test_db_session)
|
||||
|
||||
success = enricher.enrich_by_ticker("NOTFOUND")
|
||||
assert success is False
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Tests for trade loader (ETL).
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from pote.db.models import Official, Trade
|
||||
from pote.ingestion.trade_loader import TradeLoader
|
||||
|
||||
|
||||
def test_ingest_transactions_from_fixture(test_db_session):
|
||||
"""Test ingesting transactions from fixture file."""
|
||||
# Load fixture
|
||||
fixture_path = Path(__file__).parent / "fixtures" / "sample_house_watcher.json"
|
||||
with open(fixture_path) as f:
|
||||
transactions = json.load(f)
|
||||
|
||||
# Ingest
|
||||
loader = TradeLoader(test_db_session)
|
||||
counts = loader.ingest_transactions(transactions)
|
||||
|
||||
# Verify counts
|
||||
assert counts["officials"] >= 3 # Nancy, Josh, Tommy, Dan
|
||||
assert counts["securities"] >= 4 # NVDA, MSFT, AAPL, TSLA, GOOGL
|
||||
assert counts["trades"] == 5
|
||||
|
||||
# Verify data in DB
|
||||
stmt = select(Official).where(Official.name == "Nancy Pelosi")
|
||||
pelosi = test_db_session.scalars(stmt).first()
|
||||
assert pelosi is not None
|
||||
assert pelosi.chamber == "House"
|
||||
assert pelosi.party == "Democrat"
|
||||
|
||||
# Verify trades
|
||||
stmt = select(Trade).where(Trade.official_id == pelosi.id)
|
||||
pelosi_trades = test_db_session.scalars(stmt).all()
|
||||
assert len(pelosi_trades) == 2 # NVDA and GOOGL
|
||||
|
||||
# Check one trade in detail
|
||||
nvda_trade = [t for t in pelosi_trades if t.security.ticker == "NVDA"][0]
|
||||
assert nvda_trade.transaction_date == date(2024, 1, 15)
|
||||
assert nvda_trade.filing_date == date(2024, 2, 1)
|
||||
assert nvda_trade.side == "buy"
|
||||
assert nvda_trade.value_min == Decimal("1001")
|
||||
assert nvda_trade.value_max == Decimal("15000")
|
||||
|
||||
|
||||
def test_ingest_duplicate_transaction(test_db_session):
|
||||
"""Test that duplicate transactions are not created."""
|
||||
loader = TradeLoader(test_db_session)
|
||||
|
||||
transaction = {
|
||||
"representative": "Test Official",
|
||||
"ticker": "AAPL",
|
||||
"transaction_date": "01/15/2024",
|
||||
"disclosure_date": "02/01/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"party": "Independent",
|
||||
}
|
||||
|
||||
# Ingest once
|
||||
counts1 = loader.ingest_transactions([transaction])
|
||||
assert counts1["trades"] == 1
|
||||
|
||||
# Ingest again (should detect duplicate)
|
||||
counts2 = loader.ingest_transactions([transaction])
|
||||
assert counts2["trades"] == 0 # No new trade created
|
||||
|
||||
# Verify only one trade in DB
|
||||
stmt = select(Trade)
|
||||
trades = test_db_session.scalars(stmt).all()
|
||||
assert len(trades) == 1
|
||||
|
||||
|
||||
def test_ingest_transaction_missing_ticker(test_db_session):
|
||||
"""Test that transactions without tickers are skipped."""
|
||||
loader = TradeLoader(test_db_session)
|
||||
|
||||
transaction = {
|
||||
"representative": "Test Official",
|
||||
"ticker": "", # Missing ticker
|
||||
"transaction_date": "01/15/2024",
|
||||
"disclosure_date": "02/01/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"party": "Independent",
|
||||
}
|
||||
|
||||
counts = loader.ingest_transactions([transaction])
|
||||
assert counts["trades"] == 0
|
||||
|
||||
|
||||
def test_get_or_create_official_senate(test_db_session):
|
||||
"""Test creating a Senate official."""
|
||||
loader = TradeLoader(test_db_session)
|
||||
|
||||
transaction = {
|
||||
"representative": "Test Senator",
|
||||
"ticker": "AAPL",
|
||||
"transaction_date": "01/15/2024",
|
||||
"disclosure_date": "02/01/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "Senate",
|
||||
"party": "Republican",
|
||||
}
|
||||
|
||||
loader.ingest_transactions([transaction])
|
||||
|
||||
stmt = select(Official).where(Official.name == "Test Senator")
|
||||
official = test_db_session.scalars(stmt).first()
|
||||
|
||||
assert official is not None
|
||||
assert official.chamber == "Senate"
|
||||
assert official.party == "Republican"
|
||||
|
||||
|
||||
def test_multiple_trades_same_official(test_db_session):
|
||||
"""Test multiple trades for the same official."""
|
||||
loader = TradeLoader(test_db_session)
|
||||
|
||||
transactions = [
|
||||
{
|
||||
"representative": "Jane Doe",
|
||||
"ticker": "AAPL",
|
||||
"transaction_date": "01/10/2024",
|
||||
"disclosure_date": "01/25/2024",
|
||||
"transaction": "Purchase",
|
||||
"amount": "$1,001 - $15,000",
|
||||
"house": "House",
|
||||
"party": "Democrat",
|
||||
},
|
||||
{
|
||||
"representative": "Jane Doe",
|
||||
"ticker": "MSFT",
|
||||
"transaction_date": "01/15/2024",
|
||||
"disclosure_date": "01/30/2024",
|
||||
"transaction": "Sale",
|
||||
"amount": "$15,001 - $50,000",
|
||||
"house": "House",
|
||||
"party": "Democrat",
|
||||
},
|
||||
]
|
||||
|
||||
counts = loader.ingest_transactions(transactions)
|
||||
|
||||
assert counts["officials"] == 1 # Only one official created
|
||||
assert counts["trades"] == 2
|
||||
|
||||
stmt = select(Official).where(Official.name == "Jane Doe")
|
||||
official = test_db_session.scalars(stmt).first()
|
||||
|
||||
stmt = select(Trade).where(Trade.official_id == official.id)
|
||||
trades = test_db_session.scalars(stmt).all()
|
||||
|
||||
assert len(trades) == 2
|
||||
Reference in New Issue
Block a user