Add complete automation, reporting, and CI/CD system
Features Added: ============== 📧 EMAIL REPORTING SYSTEM: - EmailReporter: Send reports via SMTP (Gmail, SendGrid, custom) - ReportGenerator: Generate daily/weekly summaries with HTML/text formatting - Configurable via .env (SMTP_HOST, SMTP_PORT, etc.) - Scripts: send_daily_report.py, send_weekly_report.py 🤖 AUTOMATED RUNS: - automated_daily_run.sh: Full daily ETL pipeline + reporting - automated_weekly_run.sh: Weekly pattern analysis + reports - setup_cron.sh: Interactive cron job setup (5-minute setup) - Logs saved to ~/logs/ with automatic cleanup 🔍 HEALTH CHECKS: - health_check.py: System health monitoring - Checks: DB connection, data freshness, counts, recent alerts - JSON output for programmatic use - Exit codes for monitoring integration 🚀 CI/CD PIPELINE: - .github/workflows/ci.yml: Full CI/CD pipeline - GitHub Actions / Gitea Actions compatible - Jobs: lint & test, security scan, dependency scan, Docker build - PostgreSQL service for integration tests - 93 tests passing in CI 📚 COMPREHENSIVE DOCUMENTATION: - AUTOMATION_QUICKSTART.md: 5-minute email setup guide - docs/12_automation_and_reporting.md: Full automation guide - Updated README.md with automation links - Deployment → Production workflow guide 🛠️ IMPROVEMENTS: - All shell scripts made executable - Environment variable examples in .env.example - Report logs saved with timestamps - 30-day log retention with auto-cleanup - Health checks can be scheduled via cron WHAT THIS ENABLES: ================== After deployment, users can: 1. Set up automated daily/weekly email reports (5 min) 2. Receive HTML+text emails with: - New trades, market alerts, suspicious timing - Weekly patterns, rankings, repeat offenders 3. Monitor system health automatically 4. Run full CI/CD pipeline on every commit 5. Deploy with confidence (tests + security scans) USAGE: ====== # One-time setup (on deployed server) ./scripts/setup_cron.sh # Or manually send reports python scripts/send_daily_report.py --to user@example.com python scripts/send_weekly_report.py --to user@example.com # Check system health python scripts/health_check.py See AUTOMATION_QUICKSTART.md for full instructions. 93 tests passing | Full CI/CD | Email reports ready
This commit is contained in:
ilia
+34
-55
@@ -5,78 +5,49 @@ build-backend = "setuptools.build_meta"
|
||||
[project]
|
||||
name = "pote"
|
||||
version = "0.1.0"
|
||||
description = "Public Officials Trading Explorer – research-only transparency tool"
|
||||
description = "Public Officials Trading Explorer - research tool for congressional stock trading analysis"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = {text = "MIT"}
|
||||
authors = [
|
||||
{name = "POTE Research", email = "research@example.com"}
|
||||
]
|
||||
requires-python = ">=3.11"
|
||||
license = { text = "MIT" }
|
||||
authors = [{ name = "POTE Team" }]
|
||||
|
||||
dependencies = [
|
||||
"sqlalchemy>=2.0",
|
||||
"alembic>=1.13",
|
||||
"psycopg2-binary>=2.9",
|
||||
"alembic>=1.12",
|
||||
"pydantic>=2.0",
|
||||
"pydantic-settings>=2.0",
|
||||
"python-dotenv>=1.0",
|
||||
"requests>=2.31",
|
||||
"pandas>=2.0",
|
||||
"numpy>=1.24",
|
||||
"httpx>=0.25",
|
||||
"yfinance>=0.2.35",
|
||||
"python-dotenv>=1.0",
|
||||
"click>=8.1",
|
||||
"yfinance>=0.2",
|
||||
"psycopg2-binary>=2.9",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.4",
|
||||
"pytest-cov>=4.1",
|
||||
"pytest-asyncio>=0.21",
|
||||
"ruff>=0.1",
|
||||
"black>=23.0",
|
||||
"mypy>=1.7",
|
||||
"ipython>=8.0",
|
||||
]
|
||||
analytics = [
|
||||
"scikit-learn>=1.3",
|
||||
"matplotlib>=3.7",
|
||||
"plotly>=5.18",
|
||||
]
|
||||
api = [
|
||||
"fastapi>=0.104",
|
||||
"uvicorn[standard]>=0.24",
|
||||
"mypy>=1.5",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ["py310", "py311"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py310"
|
||||
target-version = "py311"
|
||||
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM", "RET"]
|
||||
ignore = ["E501"] # Line too long (handled by black)
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"I", # isort
|
||||
"B", # flake8-bugbear
|
||||
"C4", # flake8-comprehensions
|
||||
"UP", # pyupgrade
|
||||
]
|
||||
ignore = [
|
||||
"E501", # line too long (handled by black)
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"__init__.py" = ["F401"]
|
||||
"tests/*.py" = ["B011"] # allow assert False in tests
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ["py311"]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.10"
|
||||
python_version = "3.11"
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = false
|
||||
@@ -84,12 +55,20 @@ ignore_missing_imports = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
addopts = "-v --strict-markers --tb=short"
|
||||
markers = [
|
||||
"integration: marks tests as integration tests (require DB/network)",
|
||||
"slow: marks tests as slow",
|
||||
]
|
||||
python_files = "test_*.py"
|
||||
python_classes = "Test*"
|
||||
python_functions = "test_*"
|
||||
addopts = "-v --strict-markers"
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src/pote"]
|
||||
omit = ["*/tests/*", "*/migrations/*"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"def __repr__",
|
||||
"raise AssertionError",
|
||||
"raise NotImplementedError",
|
||||
"if __name__ == .__main__.:",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user