ilia/POTE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 3: Pattern Detection & Comparative Analysis - COMPLETE

Browse Source 
COMPLETE: Cross-official pattern detection and ranking system

New Module:
- src/pote/monitoring/pattern_detector.py: Pattern analysis engine
  * rank_officials_by_timing(): Rank all officials by suspicion
  * identify_repeat_offenders(): Find systematic offenders
  * analyze_ticker_patterns(): Per-stock suspicious patterns
  * get_sector_timing_analysis(): Sector-level analysis
  * get_party_comparison(): Democrat vs Republican comparison
  * generate_pattern_report(): Comprehensive report

Analysis Features:
- Official Rankings:
  * By average timing score
  * Suspicious trade percentage
  * Alert rates
  * Pattern classification

- Repeat Offender Detection:
  * Identifies officials with 50%+ suspicious trades
  * Historical pattern tracking
  * Systematic timing advantage detection

- Comparative Analysis:
  * Cross-party comparison
  * Sector analysis
  * Ticker-specific patterns
  * Statistical aggregations

New Script:
- scripts/generate_pattern_report.py: Comprehensive reports
  * Top 10 most suspicious officials
  * Repeat offenders list
  * Most suspiciously traded stocks
  * Sector breakdowns
  * Party comparison stats
  * Text/JSON formats

New Tests (11 total, all passing):
- test_rank_officials_by_timing
- test_identify_repeat_offenders
- test_analyze_ticker_patterns
- test_get_sector_timing_analysis
- test_get_party_comparison
- test_generate_pattern_report
- test_rank_officials_min_trades_filter
- test_empty_data_handling
- test_ranking_score_accuracy
- test_sector_stats_accuracy
- test_party_stats_completeness

Usage:
  python scripts/generate_pattern_report.py --days 365

Report Includes:
- Top suspicious officials ranked
- Repeat offenders (50%+ suspicious rate)
- Most suspiciously traded tickers
- Sector analysis
- Party comparison
- Interpretation guide

Total Test Suite: 93 tests passing ✅

ALL 3 PHASES COMPLETE!
This commit is contained in:
ilia
2025-12-15 15:23:40 -05:00
parent a52313145b
commit 2ec4a8e373
4 changed files with 919 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pote/monitoring/__init__.py
+2 -1
View File
@@ -6,6 +6,7 @@ Real-time tracking of unusual market activity.
from .alert_manager import AlertManager
from .disclosure_correlator import DisclosureCorrelator
from .market_monitor import MarketMonitor
from .pattern_detector import PatternDetector
__all__ = ["MarketMonitor", "AlertManager", "DisclosureCorrelator"]
__all__ = ["MarketMonitor", "AlertManager", "DisclosureCorrelator", "PatternDetector"]
src/pote/monitoring/pattern_detector.py
+359
View File
@@ -0,0 +1,359 @@
"""
Pattern detection across officials and stocks.
Identifies recurring suspicious behavior and trading patterns.
"""
import logging
from datetime import date, timedelta
from decimal import Decimal
from typing import Any
from sqlalchemy import and_, func
from sqlalchemy.orm import Session
from pote.db.models import MarketAlert, Official, Security, Trade
from pote.monitoring.disclosure_correlator import DisclosureCorrelator
logger = logging.getLogger(__name__)
class PatternDetector:
"""
Detect patterns in congressional trading behavior.
Identifies repeat offenders and systematic advantages.
"""
def __init__(self, session: Session):
"""Initialize pattern detector."""
self.session = session
self.correlator = DisclosureCorrelator(session)
def rank_officials_by_timing(
self, lookback_days: int = 365, min_trades: int = 3
) -> list[dict[str, Any]]:
"""
Rank officials by suspicious timing scores.
Args:
lookback_days: Days of history to analyze
min_trades: Minimum trades to include official
Returns:
List of officials ranked by avg timing score
"""
since_date = date.today() - timedelta(days=lookback_days)
# Get all officials with recent trades
officials_with_trades = (
self.session.query(
Official.id,
Official.name,
Official.chamber,
Official.party,
Official.state,
func.count(Trade.id).label("trade_count"),
)
.join(Trade)
.filter(Trade.transaction_date >= since_date)
.group_by(Official.id)
.having(func.count(Trade.id) >= min_trades)
.all()
)
logger.info(
f"Analyzing {len(officials_with_trades)} officials with {min_trades}+ trades"
)
rankings = []
for official_data in officials_with_trades:
official_id, name, chamber, party, state, trade_count = official_data
# Get timing pattern
pattern = self.correlator.get_official_timing_pattern(
official_id, lookback_days
)
if pattern["trade_count"] == 0:
continue
# Calculate percentages
alert_rate = (
pattern["trades_with_prior_alerts"] / pattern["trade_count"]
if pattern["trade_count"] > 0
else 0
)
suspicious_rate = (
pattern["suspicious_trade_count"] / pattern["trade_count"]
if pattern["trade_count"] > 0
else 0
)
rankings.append(
{
"official_id": official_id,
"name": name,
"chamber": chamber,
"party": party,
"state": state,
"trade_count": pattern["trade_count"],
"trades_with_alerts": pattern["trades_with_prior_alerts"],
"suspicious_trades": pattern["suspicious_trade_count"],
"highly_suspicious_trades": pattern["highly_suspicious_count"],
"avg_timing_score": pattern["avg_timing_score"],
"alert_rate": round(alert_rate * 100, 1),
"suspicious_rate": round(suspicious_rate * 100, 1),
"pattern": pattern["pattern"],
}
)
# Sort by average timing score (descending)
rankings.sort(key=lambda x: x["avg_timing_score"], reverse=True)
return rankings
def identify_repeat_offenders(
self, lookback_days: int = 365, min_suspicious_rate: float = 0.5
) -> list[dict[str, Any]]:
"""
Identify officials with consistent suspicious timing.
Args:
lookback_days: Days of history
min_suspicious_rate: Minimum percentage of suspicious trades
Returns:
List of repeat offenders
"""
rankings = self.rank_officials_by_timing(lookback_days, min_trades=5)
# Filter for high suspicious rates
offenders = [
r for r in rankings if r["suspicious_rate"] >= min_suspicious_rate * 100
]
logger.info(
f"Found {len(offenders)} officials with {min_suspicious_rate*100}%+ suspicious trades"
)
return offenders
def analyze_ticker_patterns(
self, lookback_days: int = 365, min_trades: int = 3
) -> list[dict[str, Any]]:
"""
Analyze which tickers show most suspicious trading patterns.
Args:
lookback_days: Days of history
min_trades: Minimum trades to include ticker
Returns:
List of tickers ranked by timing patterns
"""
since_date = date.today() - timedelta(days=lookback_days)
# Get tickers with enough trades
tickers_with_trades = (
self.session.query(
Security.ticker, func.count(Trade.id).label("trade_count")
)
.join(Trade)
.filter(Trade.transaction_date >= since_date)
.group_by(Security.ticker)
.having(func.count(Trade.id) >= min_trades)
.all()
)
logger.info(f"Analyzing {len(tickers_with_trades)} tickers")
ticker_patterns = []
for ticker, trade_count in tickers_with_trades:
analysis = self.correlator.get_ticker_timing_analysis(
ticker, lookback_days
)
if analysis["trade_count"] == 0:
continue
suspicious_rate = (
analysis["suspicious_count"] / analysis["trade_count"]
if analysis["trade_count"] > 0
else 0
)
ticker_patterns.append(
{
"ticker": ticker,
"trade_count": analysis["trade_count"],
"trades_with_alerts": analysis["trades_with_alerts"],
"suspicious_count": analysis["suspicious_count"],
"avg_timing_score": analysis["avg_timing_score"],
"suspicious_rate": round(suspicious_rate * 100, 1),
}
)
# Sort by average timing score
ticker_patterns.sort(key=lambda x: x["avg_timing_score"], reverse=True)
return ticker_patterns
def get_sector_timing_analysis(
self, lookback_days: int = 365
) -> dict[str, dict[str, Any]]:
"""
Analyze timing patterns by sector.
Args:
lookback_days: Days of history
Returns:
Dict mapping sector to timing stats
"""
since_date = date.today() - timedelta(days=lookback_days)
# Get trades grouped by sector
trades = (
self.session.query(Trade)
.join(Trade.security)
.filter(Trade.transaction_date >= since_date)
.all()
)
logger.info(f"Analyzing {len(trades)} trades by sector")
sector_stats: dict[str, dict[str, Any]] = {}
for trade in trades:
if not trade.security or not trade.security.sector:
continue
sector = trade.security.sector
if sector not in sector_stats:
sector_stats[sector] = {
"trade_count": 0,
"trades_with_alerts": 0,
"suspicious_count": 0,
"total_timing_score": 0,
}
# Analyze this trade
analysis = self.correlator.analyze_trade(trade)
sector_stats[sector]["trade_count"] += 1
sector_stats[sector]["total_timing_score"] += analysis["timing_score"]
if analysis["alert_count"] > 0:
sector_stats[sector]["trades_with_alerts"] += 1
if analysis["suspicious"]:
sector_stats[sector]["suspicious_count"] += 1
# Calculate averages
for sector, stats in sector_stats.items():
if stats["trade_count"] > 0:
stats["avg_timing_score"] = round(
stats["total_timing_score"] / stats["trade_count"], 2
)
stats["alert_rate"] = round(
stats["trades_with_alerts"] / stats["trade_count"] * 100, 1
)
stats["suspicious_rate"] = round(
stats["suspicious_count"] / stats["trade_count"] * 100, 1
)
return sector_stats
def get_party_comparison(
self, lookback_days: int = 365
) -> dict[str, dict[str, Any]]:
"""
Compare timing patterns between political parties.
Args:
lookback_days: Days of history
Returns:
Dict mapping party to timing stats
"""
rankings = self.rank_officials_by_timing(lookback_days, min_trades=1)
party_stats: dict[str, dict[str, Any]] = {}
for ranking in rankings:
party = ranking["party"]
if party not in party_stats:
party_stats[party] = {
"official_count": 0,
"total_trades": 0,
"total_suspicious": 0,
"total_timing_score": 0,
"officials": [],
}
party_stats[party]["official_count"] += 1
party_stats[party]["total_trades"] += ranking["trade_count"]
party_stats[party]["total_suspicious"] += ranking["suspicious_trades"]
party_stats[party]["total_timing_score"] += (
ranking["avg_timing_score"] * ranking["trade_count"]
)
party_stats[party]["officials"].append(ranking)
# Calculate averages
for party, stats in party_stats.items():
if stats["total_trades"] > 0:
stats["avg_timing_score"] = round(
stats["total_timing_score"] / stats["total_trades"], 2
)
stats["suspicious_rate"] = round(
stats["total_suspicious"] / stats["total_trades"] * 100, 1
)
return party_stats
def generate_pattern_report(self, lookback_days: int = 365) -> dict[str, Any]:
"""
Generate comprehensive pattern analysis report.
Args:
lookback_days: Days of history
Returns:
Complete pattern analysis
"""
logger.info(f"Generating comprehensive pattern report for last {lookback_days} days")
# Get all analyses
official_rankings = self.rank_officials_by_timing(lookback_days, min_trades=3)
repeat_offenders = self.identify_repeat_offenders(lookback_days)
ticker_patterns = self.analyze_ticker_patterns(lookback_days, min_trades=3)
sector_analysis = self.get_sector_timing_analysis(lookback_days)
party_comparison = self.get_party_comparison(lookback_days)
# Calculate summary statistics
total_officials = len(official_rankings)
total_offenders = len(repeat_offenders)
avg_timing_score = (
sum(r["avg_timing_score"] for r in official_rankings) / total_officials
if total_officials > 0
else 0
)
return {
"period_days": lookback_days,
"summary": {
"total_officials_analyzed": total_officials,
"repeat_offenders": total_offenders,
"avg_timing_score": round(avg_timing_score, 2),
},
"top_suspicious_officials": official_rankings[:10],
"repeat_offenders": repeat_offenders,
"suspicious_tickers": ticker_patterns[:10],
"sector_analysis": sector_analysis,
"party_comparison": party_comparison,
}