From be749022341b029667bf924b7656e1891f282700 Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 26 May 2026 20:22:08 -0400 Subject: [PATCH 1/2] Add homelab ops handoff doc after PR #1 merge to main. Documents LXC 236 state, cron/email setup, deploy workflow, and prioritized next steps for the follow-up agent. --- README.md | 2 + docs/HANDOFF-2026-05-27.md | 166 +++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 docs/HANDOFF-2026-05-27.md diff --git a/README.md b/README.md index 0ecf6f4..1625651 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ POTE tracks stock trading activity of government officials (starting with U.S. C **πŸ“§ Want automated reports?** See **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** for email reporting setup! +**🏠 Homelab deploy (LXC 236)?** See **[docs/HANDOFF-2026-05-27.md](docs/HANDOFF-2026-05-27.md)** for ops handoff and next steps. + ### Local Development ```bash # Install diff --git a/docs/HANDOFF-2026-05-27.md b/docs/HANDOFF-2026-05-27.md new file mode 100644 index 0000000..2d59e4c --- /dev/null +++ b/docs/HANDOFF-2026-05-27.md @@ -0,0 +1,166 @@ +# POTE homelab handoff β€” 2026-05-27 + +**Status:** Production LXC running; PR #1 merged to `main`; CI green on Gitea Actions. +**Research only β€” not investment advice.** + +--- + +## What’s live + +| Item | Value | +|------|--------| +| Host | LXC **236** `pote` @ **10.0.10.48** (pve10) | +| App | `/home/poteapp/pote` (venv, **no git clone** β€” deploy via rsync) | +| DB | PostgreSQL `pote` / `poteuser` (password rotated; in Ansible vault) | +| Data | ~55 officials, ~329 trades (30-day live ingest, May 2026) | +| SMTP | `10.0.10.132` (Mailcow), send as **`alerts@levkine.ca`** | +| Reports | **`idobkin@gmail.com`** daily 07:00, weekly Sun 08:00 | + +### Cron (`crontab -u poteapp -l`) + +| Time | Script | +|------|--------| +| 06:00 | `fetch_congressional_trades.py --days 7` | +| 06:15 | `enrich_securities.py` | +| 06:30 | `monitor_market.py --scan` | +| 07:00 | `send_daily_report.py --to idobkin@gmail.com` | +| Sun 08:00 | `send_weekly_report.py --to idobkin@gmail.com` | + +### Data source (important) + +Legacy **housestockwatcher.com** and S3 buckets are dead/blocked. Ingest uses public JSON from [congress-trading-monitor](https://github.com/kadoa-org/congress-trading-monitor) (~5000 rows cap). Override with env `POTE_HOUSE_DATA_URL` if you add another feed. + +--- + +## Repos & branches + +| Repo | Branch | Notes | +|------|--------|--------| +| **POTE** | `main` @ `git.levkin.ca/ilia/POTE` | Merged PR #1 β€” ingest, email, CI, deps | +| **ansible** | `feature/outline-setup-api` (or `master`) | Inventory, `deploy-pote.sh`, vault β€” may need merge to homelab default branch | + +Local: + +```bash +cd ~/Documents/code/POTE && git checkout main && git pull +``` + +--- + +## Quick access + +```bash +ssh root@10.0.10.48 +su - poteapp +cd pote && source venv/bin/activate + +# Logs +tail -f ~/logs/daily_report.log +tail -f ~/logs/trades.log + +# Manual run +python scripts/fetch_congressional_trades.py --days 30 +python scripts/send_daily_report.py --to idobkin@gmail.com --test-smtp +``` + +Deploy code from laptop (preserves server `.env`): + +```bash +cd ~/Documents/code/ansible +make deploy-pote +# or: RUN_FETCH=1 make deploy-pote +``` + +--- + +## Ansible / homelab inventory + +Already wired (ansible repo): + +- `inventories/production/hosts` β€” `pote` @ `.48`, VMID 236 +- `docs/guides/host-list.md` β€” LXC 236 row +- `scripts/beszel-install-agents.sh` β€” `pote-236` +- `scripts/deploy-pote.sh`, `make deploy-pote` +- `scripts/vault-update-pote.py`, `make vault-update-pote` +- `docs/guides/smtp-inventory.md` β€” POTE uses `alerts@levkine.ca` +- Vault: `vault_pote_db_password_prod`, `vault_pote_smtp_password` + +```bash +make vault-export-env +make beszel-install-agents BESZEL_ONLY=pote-236 # if agent not yet installed +``` + +--- + +## Verify after first automated day + +1. **07:00+** β€” Email in Gmail (From: `alerts@levkine.ca`, subject `POTE Daily Report - YYYY-MM-DD`). Check spam once. +2. **Logs** β€” `~/logs/daily_report.log`, `trades.log` β€” no tracebacks. +3. **DB growth** β€” trade count should tick up on weekdays: + + ```bash + su - poteapp -c 'cd pote && source venv/bin/activate && python -c " + from sqlalchemy import func, select + from pote.db import SessionLocal + from pote.db.models import Trade, Official + with SessionLocal() as s: + print(\"trades\", s.scalar(select(func.count(Trade.id)))) + print(\"officials\", s.scalar(select(func.count(Official.id)))) + "' + ``` + +--- + +## Next steps (priority order) + +### Soon + +- [ ] **Sync server to `main`** β€” `make deploy-pote` from laptop (server was rsync’d pre-merge; safe to refresh). +- [ ] **Confirm first cron email** β€” tomorrow 07:00 server time; fix via `crontab -e` / `.env` `REPORT_RECIPIENTS` if needed. +- [ ] **Beszel agent** β€” `make beszel-install-agents BESZEL_ONLY=pote-236` if not done. +- [ ] **Proxmox backup** β€” schedule backup for LXC 236 on pve10. +- [ ] **Merge ansible branch** β€” homelab inventory/deploy scripts if still on `feature/outline-setup-api`. + +### Optional enhancements + +- [ ] **Dedicated mailbox** β€” e.g. `pote@levkine.ca` in Mailcow instead of shared `alerts@` ([smtp-inventory.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/smtp-inventory.md) in ansible repo). +- [ ] **Git deploy on LXC** β€” clone `gitea@git.levkin.ca:ilia/POTE.git` + deploy key; replace rsync-only workflow. +- [ ] **UniFi DNS** β€” `mail.levkine.ca` β†’ `10.0.10.132` so `.env` can use hostname instead of raw IP ([unifi-static-dhcp.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/unifi-static-dhcp.md)). +- [ ] **Full history ingest** β€” kadoa JSON is capped; add second source (capitol-api, Clerk scrape) for backfill. +- [ ] **Kuma monitor** β€” LAN HTTP health script or SSH check (no public URL). +- [ ] **Mattermost / webhook alerts** β€” not implemented; email only today. + +### Not planned (unless you want them) + +- Public URL / Caddy vhost (LAN-only by design) +- Investment signals exposed as advice (research descriptors only) + +--- + +## Known issues / caveats + +| Topic | Detail | +|-------|--------| +| **Disclosure lag** | STOCK Act filings appear weeks after trades; reports are descriptive, not timely trading signals. | +| **Amount ranges** | Disclosure buckets only ($1k–$15k, etc.), not exact sizes. | +| **Empty tickers** | Some filings skipped when ticker missing. | +| **CI vs prod** | CI uses venv + Postgres service; prod uses host Postgres β€” both should pass after merge. | +| **Gitea deploy workflow** | `.github/workflows/deploy.yml` still references `git pull` on Proxmox; prod uses **rsync** via ansible `deploy-pote.sh`. | + +--- + +## Related docs + +| Doc | Purpose | +|-----|---------| +| [EMAIL_SETUP.md](../EMAIL_SETUP.md) | SMTP / Mailcow / levkine.ca | +| [AUTOMATION_QUICKSTART.md](../AUTOMATION_QUICKSTART.md) | Cron + reports | +| [PROXMOX_QUICKSTART.md](../PROXMOX_QUICKSTART.md) | Original LXC provisioning | +| Ansible `docs/guides/projects-handoff-2026-05-26.md` | Multi-project homelab context | +| Ansible `docs/guides/smtp-inventory.md` | Mailboxes | + +--- + +## One-line summary + +**POTE on 10.0.10.48 ingests public congressional trades daily, emails a research summary to Gmail at 07:00, and is maintained via `main` + `make deploy-pote` β€” verify tomorrow’s cron email, then Beszel, backups, and optional data-source expansion.** -- 2.49.1 From 7397334bfaa5d3a293cb4f6c38d1a5e750095481 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 12 Jul 2026 11:24:38 -0400 Subject: [PATCH 2/2] Refresh handoff next-steps and harden daily report delivery. --- docs/HANDOFF-2026-05-27.md | 28 ++++++++++------------ scripts/send_daily_report.py | 10 +++++++- src/pote/reporting/report_generator.py | 33 ++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/docs/HANDOFF-2026-05-27.md b/docs/HANDOFF-2026-05-27.md index 2d59e4c..9e7d571 100644 --- a/docs/HANDOFF-2026-05-27.md +++ b/docs/HANDOFF-2026-05-27.md @@ -111,24 +111,22 @@ make beszel-install-agents BESZEL_ONLY=pote-236 # if agent not yet installed --- -## Next steps (priority order) +**Vikunja:** [todo.levkin.ca β†’ Business β†’ POTE](https://todo.levkin.ca) (`POTE`) -### Soon +## Open tasks (source of truth) -- [ ] **Sync server to `main`** β€” `make deploy-pote` from laptop (server was rsync’d pre-merge; safe to refresh). -- [ ] **Confirm first cron email** β€” tomorrow 07:00 server time; fix via `crontab -e` / `.env` `REPORT_RECIPIENTS` if needed. -- [ ] **Beszel agent** β€” `make beszel-install-agents BESZEL_ONLY=pote-236` if not done. -- [ ] **Proxmox backup** β€” schedule backup for LXC 236 on pve10. -- [ ] **Merge ansible branch** β€” homelab inventory/deploy scripts if still on `feature/outline-setup-api`. +| P | Task | Owner | Status | +|---|------|-------|--------| +| **P1** | `make deploy-pote` β€” sync server to `main` | @agent | ⏳ | +| **P1** | Confirm 07:00 cron email delivered | @agent | ⏳ | +| **P1** | Proxmox backup schedule for LXC **236** on pve10 | @you | ⏳ | +| **P3** | Dedicated `pote@levkine.ca` mailbox (vs shared `alerts@`) | @you | optional / low | +| **P3** | Git deploy on LXC (replace rsync-only) | @agent | optional / low | +| **P3** | Kuma LAN health monitor | @agent | optional / low | +| **P3** | Mattermost / webhook alerts (email only today) | @agent | optional / low | +| **P3** | Full history ingest (second data source beyond kadoa cap) | @agent | optional / low | -### Optional enhancements - -- [ ] **Dedicated mailbox** β€” e.g. `pote@levkine.ca` in Mailcow instead of shared `alerts@` ([smtp-inventory.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/smtp-inventory.md) in ansible repo). -- [ ] **Git deploy on LXC** β€” clone `gitea@git.levkin.ca:ilia/POTE.git` + deploy key; replace rsync-only workflow. -- [ ] **UniFi DNS** β€” `mail.levkine.ca` β†’ `10.0.10.132` so `.env` can use hostname instead of raw IP ([unifi-static-dhcp.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/unifi-static-dhcp.md)). -- [ ] **Full history ingest** β€” kadoa JSON is capped; add second source (capitol-api, Clerk scrape) for backfill. -- [ ] **Kuma monitor** β€” LAN HTTP health script or SSH check (no public URL). -- [ ] **Mattermost / webhook alerts** β€” not implemented; email only today. +**Closed 2026-07-11:** Beszel agent on pote-236 (`beszel-agent` active). ### Not planned (unless you want them) diff --git a/scripts/send_daily_report.py b/scripts/send_daily_report.py index 15c7994..51bbae5 100644 --- a/scripts/send_daily_report.py +++ b/scripts/send_daily_report.py @@ -43,6 +43,12 @@ def main(): action="store_true", help="Test SMTP connection before sending", ) + parser.add_argument( + "--lookback-days", + type=int, + default=1, + help="Include trades filed in the last N days ending on the report date (default: 1)", + ) parser.add_argument( "--save-to-file", help="Also save report to this file path", @@ -81,7 +87,9 @@ def main(): logger.info(f"Generating daily report for {report_date or date.today()}...") with get_session() as session: generator = ReportGenerator(session) - report_data = generator.generate_daily_summary(report_date) + report_data = generator.generate_daily_summary( + report_date, lookback_days=args.lookback_days + ) # Format as text and HTML text_body = generator.format_as_text(report_data, "daily") diff --git a/src/pote/reporting/report_generator.py b/src/pote/reporting/report_generator.py index 698e5fe..096f3cc 100644 --- a/src/pote/reporting/report_generator.py +++ b/src/pote/reporting/report_generator.py @@ -27,13 +27,15 @@ class ReportGenerator: self.detector = PatternDetector(session) def generate_daily_summary( - self, report_date: Optional[date] = None + self, report_date: Optional[date] = None, *, lookback_days: int = 1 ) -> Dict[str, Any]: """ Generate a daily summary report. Args: report_date: Date to generate report for (defaults to today) + lookback_days: Include trades filed in the last N days ending on report_date + (defaults to 1, meaning only filings on report_date). Returns: Dictionary containing report data @@ -41,12 +43,19 @@ class ReportGenerator: if report_date is None: report_date = date.today() + if lookback_days < 1: + raise ValueError("lookback_days must be >= 1") + start_of_day = datetime.combine(report_date, datetime.min.time()) end_of_day = datetime.combine(report_date, datetime.max.time()) - # Count new trades filed today + filing_start_date = report_date - timedelta(days=lookback_days - 1) + + # Trades filed within the lookback window (inclusive) new_trades = ( - self.session.query(Trade).filter(Trade.filing_date == report_date).all() + self.session.query(Trade) + .filter(Trade.filing_date >= filing_start_date, Trade.filing_date <= report_date) + .all() ) # Count market alerts today @@ -71,6 +80,8 @@ class ReportGenerator: return { "date": report_date, + "filing_start_date": filing_start_date, + "lookback_days": lookback_days, "new_trades_count": len(new_trades), "new_trades": [ { @@ -173,13 +184,20 @@ class ReportGenerator: def _format_daily_text(self, data: Dict[str, Any]) -> str: """Format daily report as plain text.""" + if data.get("lookback_days", 1) > 1: + trades_label = ( + f" β€’ Trades Filed (last {data['lookback_days']} days): {data['new_trades_count']}" + ) + else: + trades_label = f" β€’ New Trades Filed: {data['new_trades_count']}" + lines = [ "=" * 70, f"POTE DAILY REPORT - {data['date']}", "=" * 70, "", "πŸ“Š SUMMARY", - f" β€’ New Trades Filed: {data['new_trades_count']}", + trades_label, f" β€’ Market Alerts: {data['market_alerts_count']}", f" β€’ Critical Alerts (β‰₯7 severity): {data['critical_alerts_count']}", f" β€’ Suspicious Timing Trades: {data['suspicious_trades_count']}", @@ -287,6 +305,11 @@ class ReportGenerator: def _format_daily_html(self, data: Dict[str, Any]) -> str: """Format daily report as HTML.""" + if data.get("lookback_days", 1) > 1: + new_trades_label = f"Trades Filed (last {data['lookback_days']} days):" + else: + new_trades_label = "New Trades:" + html = f""" @@ -307,7 +330,7 @@ class ReportGenerator:

πŸ“Š Summary

-
New Trades: {data['new_trades_count']}
+
{new_trades_label} {data['new_trades_count']}
Market Alerts: {data['market_alerts_count']}
Critical Alerts: {data['critical_alerts_count']}
Suspicious Trades: {data['suspicious_trades_count']}
-- 2.49.1