Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ilia
|
d27f791657 | ||
|
ilia
|
7b900518fa | ||
|
ilia
|
8110c5949d | ||
|
ilia
|
fc510f2b2c | ||
|
ilia
|
9cb05ddf77 | ||
|
ilia
|
1f4e9c075a | ||
|
ilia
|
a95429509f | ||
|
ilia
|
169f28363b | ||
|
ilia
|
b383f9dd8d | ||
|
ilia
|
a8757fd6f1 | ||
|
ilia
|
3950867dae | ||
|
ilia
|
1bce7581e5 | ||
|
ilia
|
b9a2e1011f | ||
|
ilia
|
a11108838d | ||
|
ilia
|
2ee601c198 | ||
|
ilia
|
367d76eb9d |
@@ -0,0 +1,83 @@
|
||||
---
|
||||
# Homelab CI — Python lane (git-ci-01) + secret scan (git-ci-02)
|
||||
# Skip: @skipci in branch name or commit message
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, main]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
skip-ci-check:
|
||||
runs-on: [homelab, self-hosted, linux]
|
||||
container:
|
||||
image: node:20-bookworm
|
||||
outputs:
|
||||
should-skip: ${{ steps.check.outputs.skip }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- id: check
|
||||
run: |
|
||||
SKIP=0
|
||||
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
|
||||
MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-$(git log -1 --pretty=%B 2>/dev/null || true)}"
|
||||
echo "$BRANCH" "$MSG" | grep -qi '@skipci' && SKIP=1
|
||||
echo "skip=$SKIP" >> $GITHUB_OUTPUT
|
||||
|
||||
python-ci:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, python]
|
||||
container:
|
||||
# node image: actions/checkout@v4 needs Node; install python3 in-job
|
||||
image: node:20-bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python tooling
|
||||
run: |
|
||||
apt-get update -qq
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv
|
||||
python3 -m pip install --upgrade pip --break-system-packages
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt --break-system-packages; fi
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt --break-system-packages; fi
|
||||
pip install bandit pip-audit ruff --break-system-packages
|
||||
|
||||
- name: Ruff lint
|
||||
run: ruff check . || true
|
||||
|
||||
- name: Bandit (advisory)
|
||||
run: bandit -r . -q || true
|
||||
|
||||
- name: pip-audit (advisory)
|
||||
run: pip-audit -r requirements.txt 2>/dev/null || pip-audit 2>/dev/null || true
|
||||
|
||||
- name: Pytest
|
||||
run: |
|
||||
if [ -d tests ] || ls test_*.py *_test.py 2>/dev/null; then
|
||||
pip install pytest --break-system-packages
|
||||
pytest -q || true
|
||||
else
|
||||
echo "No tests found — skip"
|
||||
fi
|
||||
|
||||
secret-scan:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, heavy]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Gitleaks
|
||||
run: |
|
||||
extra=""
|
||||
if [ -f .gitleaks.toml ]; then
|
||||
extra="--config /repo/.gitleaks.toml"
|
||||
fi
|
||||
docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \
|
||||
detect --source /repo --no-banner --redact ${extra}
|
||||
@@ -0,0 +1,28 @@
|
||||
# Homelab bootstrap — gitleaks allowlist (tests, examples, placeholders)
|
||||
#
|
||||
# IMPORTANT: `useDefault = true` is required — without it gitleaks loads ONLY
|
||||
# this file (title + allowlist) with ZERO detection rules, so it would never
|
||||
# flag a real secret. Fixed 2026-07 (security-hardening track); if you're
|
||||
# re-pushing this template to a repo that already had the old version, that
|
||||
# repo's secret scanning was a no-op until this lands.
|
||||
title = "homelab gitea bootstrap"
|
||||
|
||||
[extend]
|
||||
useDefault = true
|
||||
|
||||
[allowlist]
|
||||
description = "Test fixtures and example configs are not production secrets"
|
||||
paths = [
|
||||
'''(?i).*\.test\.(ts|tsx|js|jsx|py)$''',
|
||||
'''(?i).*\.spec\.(ts|tsx|js|jsx)$''',
|
||||
'''(?i).*/tests/.*''',
|
||||
'''(?i).*/__tests__/.*''',
|
||||
'''(?i).*\.example\.(yml|yaml|env|json|toml)$''',
|
||||
'''(?i).*vault\.example\.(yml|yaml)$''',
|
||||
'''(?i).*\.env\.example$''',
|
||||
]
|
||||
regexes = [
|
||||
'''(?i)(invalid|fake|dummy|placeholder|example|changeme|change_me|not-a-real)''',
|
||||
'''(?i)sk-or-invalid''',
|
||||
'''(?i)msk-or-invalid''',
|
||||
]
|
||||
@@ -30,8 +30,6 @@ POTE tracks stock trading activity of government officials (starting with U.S. C
|
||||
|
||||
**📧 Want automated reports?** See **[AUTOMATION_QUICKSTART.md](AUTOMATION_QUICKSTART.md)** for email reporting setup!
|
||||
|
||||
**🏠 Homelab deploy (LXC 236)?** See **[docs/HANDOFF-2026-05-27.md](docs/HANDOFF-2026-05-27.md)** for ops handoff and next steps.
|
||||
|
||||
### Local Development
|
||||
```bash
|
||||
# Install
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
# POTE homelab handoff — 2026-05-27
|
||||
|
||||
**Status:** Production LXC running; PR #1 merged to `main`; CI green on Gitea Actions.
|
||||
**Research only — not investment advice.**
|
||||
|
||||
---
|
||||
|
||||
## What’s live
|
||||
|
||||
| Item | Value |
|
||||
|------|--------|
|
||||
| Host | LXC **236** `pote` @ **10.0.10.48** (pve10) |
|
||||
| App | `/home/poteapp/pote` (venv, **no git clone** — deploy via rsync) |
|
||||
| DB | PostgreSQL `pote` / `poteuser` (password rotated; in Ansible vault) |
|
||||
| Data | ~55 officials, ~329 trades (30-day live ingest, May 2026) |
|
||||
| SMTP | `10.0.10.132` (Mailcow), send as **`alerts@levkine.ca`** |
|
||||
| Reports | **`idobkin@gmail.com`** daily 07:00, weekly Sun 08:00 |
|
||||
|
||||
### Cron (`crontab -u poteapp -l`)
|
||||
|
||||
| Time | Script |
|
||||
|------|--------|
|
||||
| 06:00 | `fetch_congressional_trades.py --days 7` |
|
||||
| 06:15 | `enrich_securities.py` |
|
||||
| 06:30 | `monitor_market.py --scan` |
|
||||
| 07:00 | `send_daily_report.py --to idobkin@gmail.com` |
|
||||
| Sun 08:00 | `send_weekly_report.py --to idobkin@gmail.com` |
|
||||
|
||||
### Data source (important)
|
||||
|
||||
Legacy **housestockwatcher.com** and S3 buckets are dead/blocked. Ingest uses public JSON from [congress-trading-monitor](https://github.com/kadoa-org/congress-trading-monitor) (~5000 rows cap). Override with env `POTE_HOUSE_DATA_URL` if you add another feed.
|
||||
|
||||
---
|
||||
|
||||
## Repos & branches
|
||||
|
||||
| Repo | Branch | Notes |
|
||||
|------|--------|--------|
|
||||
| **POTE** | `main` @ `git.levkin.ca/ilia/POTE` | Merged PR #1 — ingest, email, CI, deps |
|
||||
| **ansible** | `feature/outline-setup-api` (or `master`) | Inventory, `deploy-pote.sh`, vault — may need merge to homelab default branch |
|
||||
|
||||
Local:
|
||||
|
||||
```bash
|
||||
cd ~/Documents/code/POTE && git checkout main && git pull
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick access
|
||||
|
||||
```bash
|
||||
ssh root@10.0.10.48
|
||||
su - poteapp
|
||||
cd pote && source venv/bin/activate
|
||||
|
||||
# Logs
|
||||
tail -f ~/logs/daily_report.log
|
||||
tail -f ~/logs/trades.log
|
||||
|
||||
# Manual run
|
||||
python scripts/fetch_congressional_trades.py --days 30
|
||||
python scripts/send_daily_report.py --to idobkin@gmail.com --test-smtp
|
||||
```
|
||||
|
||||
Deploy code from laptop (preserves server `.env`):
|
||||
|
||||
```bash
|
||||
cd ~/Documents/code/ansible
|
||||
make deploy-pote
|
||||
# or: RUN_FETCH=1 make deploy-pote
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ansible / homelab inventory
|
||||
|
||||
Already wired (ansible repo):
|
||||
|
||||
- `inventories/production/hosts` — `pote` @ `.48`, VMID 236
|
||||
- `docs/guides/host-list.md` — LXC 236 row
|
||||
- `scripts/beszel-install-agents.sh` — `pote-236`
|
||||
- `scripts/deploy-pote.sh`, `make deploy-pote`
|
||||
- `scripts/vault-update-pote.py`, `make vault-update-pote`
|
||||
- `docs/guides/smtp-inventory.md` — POTE uses `alerts@levkine.ca`
|
||||
- Vault: `vault_pote_db_password_prod`, `vault_pote_smtp_password`
|
||||
|
||||
```bash
|
||||
make vault-export-env
|
||||
make beszel-install-agents BESZEL_ONLY=pote-236 # if agent not yet installed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verify after first automated day
|
||||
|
||||
1. **07:00+** — Email in Gmail (From: `alerts@levkine.ca`, subject `POTE Daily Report - YYYY-MM-DD`). Check spam once.
|
||||
2. **Logs** — `~/logs/daily_report.log`, `trades.log` — no tracebacks.
|
||||
3. **DB growth** — trade count should tick up on weekdays:
|
||||
|
||||
```bash
|
||||
su - poteapp -c 'cd pote && source venv/bin/activate && python -c "
|
||||
from sqlalchemy import func, select
|
||||
from pote.db import SessionLocal
|
||||
from pote.db.models import Trade, Official
|
||||
with SessionLocal() as s:
|
||||
print(\"trades\", s.scalar(select(func.count(Trade.id))))
|
||||
print(\"officials\", s.scalar(select(func.count(Official.id))))
|
||||
"'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Next steps (priority order)
|
||||
|
||||
### Soon
|
||||
|
||||
- [ ] **Sync server to `main`** — `make deploy-pote` from laptop (server was rsync’d pre-merge; safe to refresh).
|
||||
- [ ] **Confirm first cron email** — tomorrow 07:00 server time; fix via `crontab -e` / `.env` `REPORT_RECIPIENTS` if needed.
|
||||
- [ ] **Beszel agent** — `make beszel-install-agents BESZEL_ONLY=pote-236` if not done.
|
||||
- [ ] **Proxmox backup** — schedule backup for LXC 236 on pve10.
|
||||
- [ ] **Merge ansible branch** — homelab inventory/deploy scripts if still on `feature/outline-setup-api`.
|
||||
|
||||
### Optional enhancements
|
||||
|
||||
- [ ] **Dedicated mailbox** — e.g. `pote@levkine.ca` in Mailcow instead of shared `alerts@` ([smtp-inventory.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/smtp-inventory.md) in ansible repo).
|
||||
- [ ] **Git deploy on LXC** — clone `gitea@git.levkin.ca:ilia/POTE.git` + deploy key; replace rsync-only workflow.
|
||||
- [ ] **UniFi DNS** — `mail.levkine.ca` → `10.0.10.132` so `.env` can use hostname instead of raw IP ([unifi-static-dhcp.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/unifi-static-dhcp.md)).
|
||||
- [ ] **Full history ingest** — kadoa JSON is capped; add second source (capitol-api, Clerk scrape) for backfill.
|
||||
- [ ] **Kuma monitor** — LAN HTTP health script or SSH check (no public URL).
|
||||
- [ ] **Mattermost / webhook alerts** — not implemented; email only today.
|
||||
|
||||
### Not planned (unless you want them)
|
||||
|
||||
- Public URL / Caddy vhost (LAN-only by design)
|
||||
- Investment signals exposed as advice (research descriptors only)
|
||||
|
||||
---
|
||||
|
||||
## Known issues / caveats
|
||||
|
||||
| Topic | Detail |
|
||||
|-------|--------|
|
||||
| **Disclosure lag** | STOCK Act filings appear weeks after trades; reports are descriptive, not timely trading signals. |
|
||||
| **Amount ranges** | Disclosure buckets only ($1k–$15k, etc.), not exact sizes. |
|
||||
| **Empty tickers** | Some filings skipped when ticker missing. |
|
||||
| **CI vs prod** | CI uses venv + Postgres service; prod uses host Postgres — both should pass after merge. |
|
||||
| **Gitea deploy workflow** | `.github/workflows/deploy.yml` still references `git pull` on Proxmox; prod uses **rsync** via ansible `deploy-pote.sh`. |
|
||||
|
||||
---
|
||||
|
||||
## Related docs
|
||||
|
||||
| Doc | Purpose |
|
||||
|-----|---------|
|
||||
| [EMAIL_SETUP.md](../EMAIL_SETUP.md) | SMTP / Mailcow / levkine.ca |
|
||||
| [AUTOMATION_QUICKSTART.md](../AUTOMATION_QUICKSTART.md) | Cron + reports |
|
||||
| [PROXMOX_QUICKSTART.md](../PROXMOX_QUICKSTART.md) | Original LXC provisioning |
|
||||
| Ansible `docs/guides/projects-handoff-2026-05-26.md` | Multi-project homelab context |
|
||||
| Ansible `docs/guides/smtp-inventory.md` | Mailboxes |
|
||||
|
||||
---
|
||||
|
||||
## One-line summary
|
||||
|
||||
**POTE on 10.0.10.48 ingests public congressional trades daily, emails a research summary to Gmail at 07:00, and is maintained via `main` + `make deploy-pote` — verify tomorrow’s cron email, then Beszel, backups, and optional data-source expansion.**
|
||||
Reference in New Issue
Block a user