Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ilia
|
7eac9afad1 |
@@ -1,84 +0,0 @@
|
||||
---
|
||||
# ci-sync: 2026-05-30T02:31:20Z
|
||||
# Homelab CI — Python lane (git-ci-01) + secret scan (git-ci-02)
|
||||
# Skip: @skipci in branch name or commit message
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, main]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
skip-ci-check:
|
||||
runs-on: [homelab, self-hosted, linux]
|
||||
container:
|
||||
image: node:20-bookworm
|
||||
outputs:
|
||||
should-skip: ${{ steps.check.outputs.skip }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- id: check
|
||||
run: |
|
||||
SKIP=0
|
||||
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
|
||||
MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-$(git log -1 --pretty=%B 2>/dev/null || true)}"
|
||||
echo "$BRANCH" "$MSG" | grep -qi '@skipci' && SKIP=1
|
||||
echo "skip=$SKIP" >> $GITHUB_OUTPUT
|
||||
|
||||
python-ci:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, python]
|
||||
container:
|
||||
# node image: actions/checkout@v4 needs Node; install python3 in-job
|
||||
image: node:20-bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python tooling
|
||||
run: |
|
||||
apt-get update -qq
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv
|
||||
python3 -m pip install --upgrade pip --break-system-packages
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt --break-system-packages; fi
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt --break-system-packages; fi
|
||||
pip install bandit pip-audit ruff --break-system-packages
|
||||
|
||||
- name: Ruff lint
|
||||
run: ruff check . || true
|
||||
|
||||
- name: Bandit (advisory)
|
||||
run: bandit -r . -q || true
|
||||
|
||||
- name: pip-audit (advisory)
|
||||
run: pip-audit -r requirements.txt 2>/dev/null || pip-audit 2>/dev/null || true
|
||||
|
||||
- name: Pytest
|
||||
run: |
|
||||
if [ -d tests ] || ls test_*.py *_test.py 2>/dev/null; then
|
||||
pip install pytest --break-system-packages
|
||||
pytest -q || true
|
||||
else
|
||||
echo "No tests found — skip"
|
||||
fi
|
||||
|
||||
secret-scan:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: [homelab, self-hosted, linux, heavy]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Gitleaks
|
||||
run: |
|
||||
extra=""
|
||||
if [ -f .gitleaks.toml ]; then
|
||||
extra="--config /repo/.gitleaks.toml"
|
||||
fi
|
||||
docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \
|
||||
detect --source /repo --no-banner --redact ${extra}
|
||||
@@ -1,30 +0,0 @@
|
||||
# Homelab bootstrap — gitleaks allowlist (tests, examples, placeholders)
|
||||
#
|
||||
# IMPORTANT: `useDefault = true` is required — without it gitleaks loads ONLY
|
||||
# this file (title + allowlist) with ZERO detection rules, so it would never
|
||||
# flag a real secret. Fixed 2026-07 (security-hardening track); if you're
|
||||
# re-pushing this template to a repo that already had the old version, that
|
||||
# repo's secret scanning was a no-op until this lands.
|
||||
title = "homelab gitea bootstrap"
|
||||
|
||||
[extend]
|
||||
useDefault = true
|
||||
|
||||
[allowlist]
|
||||
description = "Test fixtures and example configs are not production secrets"
|
||||
paths = [
|
||||
'''(?i).*\.test\.(ts|tsx|js|jsx|py)$''',
|
||||
'''(?i).*\.spec\.(ts|tsx|js|jsx)$''',
|
||||
'''(?i).*/tests/.*''',
|
||||
'''(?i).*/__tests__/.*''',
|
||||
'''(?i).*\.example\.(yml|yaml|env|json|toml)$''',
|
||||
'''(?i).*vault\.example\.(yml|yaml)$''',
|
||||
'''(?i).*\.env\.example$''',
|
||||
'''(?i)CUSTOMIZATION_CHECKLIST\.md$''',
|
||||
]
|
||||
regexes = [
|
||||
'''(?i)(invalid|fake|dummy|placeholder|example|changeme|change_me|not-a-real)''',
|
||||
'''(?i)sk-or-invalid''',
|
||||
'''(?i)msk-or-invalid''',
|
||||
'''(?i)your_ssh_private_key_here''',
|
||||
]
|
||||
+15
-12
@@ -111,21 +111,24 @@ make beszel-install-agents BESZEL_ONLY=pote-236 # if agent not yet installed
|
||||
|
||||
---
|
||||
|
||||
**Vikunja:** [todo.levkin.ca → Business → POTE](https://todo.levkin.ca) (`POTE`)
|
||||
## Next steps (priority order)
|
||||
|
||||
## Open tasks (source of truth)
|
||||
### Soon
|
||||
|
||||
| P | Task | Owner | Status |
|
||||
|---|------|-------|--------|
|
||||
| **P1** | Proxmox backup schedule for LXC **236** on pve10 | @you | ⏳ |
|
||||
| **P3** | Dedicated `pote@levkine.ca` mailbox (vs shared `alerts@`) | @you | optional / low |
|
||||
| **P3** | Git deploy on LXC (replace rsync-only) | @agent | optional / low |
|
||||
| **P3** | Kuma LAN health monitor | @agent | optional / low |
|
||||
| **P3** | Mattermost / webhook alerts (email only today) | @agent | optional / low |
|
||||
| **P3** | Full history ingest (second data source beyond kadoa cap) | @agent | optional / low |
|
||||
- [ ] **Sync server to `main`** — `make deploy-pote` from laptop (server was rsync’d pre-merge; safe to refresh).
|
||||
- [ ] **Confirm first cron email** — tomorrow 07:00 server time; fix via `crontab -e` / `.env` `REPORT_RECIPIENTS` if needed.
|
||||
- [ ] **Beszel agent** — `make beszel-install-agents BESZEL_ONLY=pote-236` if not done.
|
||||
- [ ] **Proxmox backup** — schedule backup for LXC 236 on pve10.
|
||||
- [ ] **Merge ansible branch** — homelab inventory/deploy scripts if still on `feature/outline-setup-api`.
|
||||
|
||||
**Closed 2026-07-11:** Beszel agent on pote-236 (`beszel-agent` active).
|
||||
**Closed 2026-07-12:** `make deploy-pote` synced to LXC 236; 07:00 cron email delivered (`idobkin@gmail.com`, log OK). DB ~1129 trades / 80 officials.
|
||||
### Optional enhancements
|
||||
|
||||
- [ ] **Dedicated mailbox** — e.g. `pote@levkine.ca` in Mailcow instead of shared `alerts@` ([smtp-inventory.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/smtp-inventory.md) in ansible repo).
|
||||
- [ ] **Git deploy on LXC** — clone `gitea@git.levkin.ca:ilia/POTE.git` + deploy key; replace rsync-only workflow.
|
||||
- [ ] **UniFi DNS** — `mail.levkine.ca` → `10.0.10.132` so `.env` can use hostname instead of raw IP ([unifi-static-dhcp.md](https://git.levkin.ca/ilia/ansible/src/branch/master/docs/guides/unifi-static-dhcp.md)).
|
||||
- [ ] **Full history ingest** — kadoa JSON is capped; add second source (capitol-api, Clerk scrape) for backfill.
|
||||
- [ ] **Kuma monitor** — LAN HTTP health script or SSH check (no public URL).
|
||||
- [ ] **Mattermost / webhook alerts** — not implemented; email only today.
|
||||
|
||||
### Not planned (unless you want them)
|
||||
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time installer for the local pre-commit gitleaks hook.
|
||||
# Run once per clone: bash scripts/git-hooks/install.sh
|
||||
#
|
||||
# Respects `core.hooksPath` if you've set one (local or global) — some setups
|
||||
# point git at a hooks dir outside `.git/hooks/` (e.g. a machine-wide
|
||||
# `~/.git-hooks/`), and installing to `.git/hooks/` in that case would be a
|
||||
# silent no-op. If an existing hook is already at that path, this chains to
|
||||
# it so nothing already relying on it breaks.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
HOOKS_DIR="$(git config --get core.hooksPath || true)"
|
||||
if [ -z "$HOOKS_DIR" ]; then
|
||||
HOOKS_DIR=".git/hooks"
|
||||
elif [[ "$HOOKS_DIR" != /* ]]; then
|
||||
HOOKS_DIR="$REPO_ROOT/$HOOKS_DIR"
|
||||
fi
|
||||
mkdir -p "$HOOKS_DIR"
|
||||
|
||||
TARGET="$HOOKS_DIR/pre-commit"
|
||||
if [ -f "$TARGET" ] && ! grep -q "gitleaks" "$TARGET" 2>/dev/null; then
|
||||
echo "⚠️ Existing pre-commit hook found at $TARGET that isn't ours — chaining instead of overwriting."
|
||||
CHAINED="$HOOKS_DIR/pre-commit.d-gitleaks"
|
||||
cp scripts/git-hooks/pre-commit "$CHAINED"
|
||||
chmod +x "$CHAINED"
|
||||
if ! grep -q "pre-commit.d-gitleaks" "$TARGET" 2>/dev/null; then
|
||||
printf '\n# Added by levkinops ansible repo (scripts/git-hooks/install.sh)\n"%s"\n' "$CHAINED" >> "$TARGET"
|
||||
fi
|
||||
else
|
||||
cp scripts/git-hooks/pre-commit "$TARGET"
|
||||
chmod +x "$TARGET"
|
||||
fi
|
||||
|
||||
echo "✓ Installed pre-commit gitleaks hook → $TARGET"
|
||||
if [ "$HOOKS_DIR" != ".git/hooks" ] && [ "$HOOKS_DIR" != "$REPO_ROOT/.git/hooks" ]; then
|
||||
echo " (using core.hooksPath=$HOOKS_DIR — applies to every repo that shares this hooksPath)"
|
||||
fi
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo " Note: gitleaks isn't installed locally yet. Install it for the hook to actually run:"
|
||||
echo " macOS: brew install gitleaks"
|
||||
echo " Linux: see https://github.com/gitleaks/gitleaks#installing"
|
||||
echo " Until then this hook is a no-op locally (CI still scans every push)."
|
||||
fi
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Local pre-commit secret scan — mirrors the `secret-scanning` CI job (gitleaks)
|
||||
# so leaked secrets are caught before they ever leave your machine, not just at
|
||||
# CI time. Installed via `make install-git-hooks`.
|
||||
#
|
||||
# Uses the same .gitleaks.toml allowlist as CI. Only scans staged content.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "⚠️ gitleaks not installed locally — skipping local secret scan."
|
||||
echo " Install: brew install gitleaks (CI will still catch it either way)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "🔐 Running gitleaks on staged changes..."
|
||||
if ! gitleaks protect --staged --config .gitleaks.toml --no-banner --redact; then
|
||||
echo ""
|
||||
echo "❌ gitleaks found a potential secret in your staged changes."
|
||||
echo " Fix it, or if it's a false positive, add an allowlist entry to .gitleaks.toml."
|
||||
echo " Bypass (not recommended): git commit --no-verify"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ gitleaks: no secrets found in staged changes"
|
||||
@@ -43,12 +43,6 @@ def main():
|
||||
action="store_true",
|
||||
help="Test SMTP connection before sending",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lookback-days",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Include trades filed in the last N days ending on the report date (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-to-file",
|
||||
help="Also save report to this file path",
|
||||
@@ -87,9 +81,7 @@ def main():
|
||||
logger.info(f"Generating daily report for {report_date or date.today()}...")
|
||||
with get_session() as session:
|
||||
generator = ReportGenerator(session)
|
||||
report_data = generator.generate_daily_summary(
|
||||
report_date, lookback_days=args.lookback_days
|
||||
)
|
||||
report_data = generator.generate_daily_summary(report_date)
|
||||
|
||||
# Format as text and HTML
|
||||
text_body = generator.format_as_text(report_data, "daily")
|
||||
|
||||
@@ -27,15 +27,13 @@ class ReportGenerator:
|
||||
self.detector = PatternDetector(session)
|
||||
|
||||
def generate_daily_summary(
|
||||
self, report_date: Optional[date] = None, *, lookback_days: int = 1
|
||||
self, report_date: Optional[date] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate a daily summary report.
|
||||
|
||||
Args:
|
||||
report_date: Date to generate report for (defaults to today)
|
||||
lookback_days: Include trades filed in the last N days ending on report_date
|
||||
(defaults to 1, meaning only filings on report_date).
|
||||
|
||||
Returns:
|
||||
Dictionary containing report data
|
||||
@@ -43,19 +41,12 @@ class ReportGenerator:
|
||||
if report_date is None:
|
||||
report_date = date.today()
|
||||
|
||||
if lookback_days < 1:
|
||||
raise ValueError("lookback_days must be >= 1")
|
||||
|
||||
start_of_day = datetime.combine(report_date, datetime.min.time())
|
||||
end_of_day = datetime.combine(report_date, datetime.max.time())
|
||||
|
||||
filing_start_date = report_date - timedelta(days=lookback_days - 1)
|
||||
|
||||
# Trades filed within the lookback window (inclusive)
|
||||
# Count new trades filed today
|
||||
new_trades = (
|
||||
self.session.query(Trade)
|
||||
.filter(Trade.filing_date >= filing_start_date, Trade.filing_date <= report_date)
|
||||
.all()
|
||||
self.session.query(Trade).filter(Trade.filing_date == report_date).all()
|
||||
)
|
||||
|
||||
# Count market alerts today
|
||||
@@ -80,8 +71,6 @@ class ReportGenerator:
|
||||
|
||||
return {
|
||||
"date": report_date,
|
||||
"filing_start_date": filing_start_date,
|
||||
"lookback_days": lookback_days,
|
||||
"new_trades_count": len(new_trades),
|
||||
"new_trades": [
|
||||
{
|
||||
@@ -184,20 +173,13 @@ class ReportGenerator:
|
||||
|
||||
def _format_daily_text(self, data: Dict[str, Any]) -> str:
|
||||
"""Format daily report as plain text."""
|
||||
if data.get("lookback_days", 1) > 1:
|
||||
trades_label = (
|
||||
f" • Trades Filed (last {data['lookback_days']} days): {data['new_trades_count']}"
|
||||
)
|
||||
else:
|
||||
trades_label = f" • New Trades Filed: {data['new_trades_count']}"
|
||||
|
||||
lines = [
|
||||
"=" * 70,
|
||||
f"POTE DAILY REPORT - {data['date']}",
|
||||
"=" * 70,
|
||||
"",
|
||||
"📊 SUMMARY",
|
||||
trades_label,
|
||||
f" • New Trades Filed: {data['new_trades_count']}",
|
||||
f" • Market Alerts: {data['market_alerts_count']}",
|
||||
f" • Critical Alerts (≥7 severity): {data['critical_alerts_count']}",
|
||||
f" • Suspicious Timing Trades: {data['suspicious_trades_count']}",
|
||||
@@ -305,11 +287,6 @@ class ReportGenerator:
|
||||
|
||||
def _format_daily_html(self, data: Dict[str, Any]) -> str:
|
||||
"""Format daily report as HTML."""
|
||||
if data.get("lookback_days", 1) > 1:
|
||||
new_trades_label = f"Trades Filed (last {data['lookback_days']} days):"
|
||||
else:
|
||||
new_trades_label = "New Trades:"
|
||||
|
||||
html = f"""
|
||||
<html>
|
||||
<head>
|
||||
@@ -330,7 +307,7 @@ class ReportGenerator:
|
||||
|
||||
<div class="summary">
|
||||
<h2>📊 Summary</h2>
|
||||
<div class="stat"><strong>{new_trades_label}</strong> {data['new_trades_count']}</div>
|
||||
<div class="stat"><strong>New Trades:</strong> {data['new_trades_count']}</div>
|
||||
<div class="stat"><strong>Market Alerts:</strong> {data['market_alerts_count']}</div>
|
||||
<div class="stat"><strong>Critical Alerts:</strong> {data['critical_alerts_count']}</div>
|
||||
<div class="stat"><strong>Suspicious Trades:</strong> {data['suspicious_trades_count']}</div>
|
||||
|
||||
Reference in New Issue
Block a user