From b2a2507615a5e7a566c49ee4070bfe651943b391 Mon Sep 17 00:00:00 2001
From: ilia <idobkin@gmail.com>
Date: Sat, 4 Apr 2026 22:03:53 -0400
Subject: [PATCH] =?UTF-8?q?fix(scripts):=20Telegram=20pipeline=20summary?=
 =?UTF-8?q?=20=E2=80=94=20timestamp=20compare,=20retries,=20diagnostics?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Normalize SQLite vs ISO discoveredAt for run-window filtering
- Retry jobs list after run; compressed JSON requests
- Cross-check GET /api/jobs/revision when list is empty
- Docs: cron at 9/13/18, pull + redeploy steps for VM

Made-with: Cursor
---
 DEPLOY_GITEA_VM_CRON_TELEGRAM.md    | 18 +++++++-
 scripts/jobber-pipeline-telegram.sh | 71 +++++++++++++++++++++++++----
 2 files changed, 78 insertions(+), 11 deletions(-)

diff --git a/DEPLOY_GITEA_VM_CRON_TELEGRAM.md b/DEPLOY_GITEA_VM_CRON_TELEGRAM.md
index 0e0ef37..f47681f 100644
--- a/DEPLOY_GITEA_VM_CRON_TELEGRAM.md
+++ b/DEPLOY_GITEA_VM_CRON_TELEGRAM.md
@@ -170,12 +170,26 @@ Fill **`TELEGRAM_BOT_TOKEN`** (from @BotFather) and **`TELEGRAM_CHAT_ID`**. For
 
 You should get one Telegram when the pipeline finishes. Optional log: append `>> /var/log/jobber-pipeline.log 2>&1` on the cron line.
 
-**4. Cron** (example: 08:00, 14:00, 20:00 host local time — `crontab -e`):
+**4. Cron** (host **local** timezone — check with `timedatectl` — `crontab -e` as root):
 
 ```cron
-0 8,14,20 * * * /usr/local/bin/jobber-pipeline-telegram.sh >> /var/log/jobber-pipeline.log 2>&1
+# 09:00, 13:00, 18:00 daily — pipeline + Telegram summary
+0 9,13,18 * * * /usr/local/bin/jobber-pipeline-telegram.sh >> /var/log/jobber-pipeline.log 2>&1
 ```
 
+Other examples: `0 8,14,20 * * *` for 08:00 / 14:00 / 20:00.
+
+**5. Pull latest code and redeploy** (on the VM, from the repo root, e.g. `/opt/Jobber`):
+
+```bash
+cd /opt/Jobber
+git fetch origin && git pull --ff-only
+install -m 755 scripts/jobber-pipeline-telegram.sh /usr/local/bin/jobber-pipeline-telegram.sh
+docker compose up -d --build
+```
+
+Wait until `curl -sf http://127.0.0.1:3005/health` succeeds before relying on cron (container needs a few seconds after start).
+
 **Security:** Never commit `/root/.jobber-cron.env` or paste bot tokens in Git. Revoke the token in BotFather if it was exposed.
 
 ### Option B2 — Minimal curl-only (no wait-for-finish)
diff --git a/scripts/jobber-pipeline-telegram.sh b/scripts/jobber-pipeline-telegram.sh
index af04727..22951d2 100755
--- a/scripts/jobber-pipeline-telegram.sh
+++ b/scripts/jobber-pipeline-telegram.sh
@@ -40,11 +40,39 @@ send_tg_html() {
 }
 
 fetch_status() {
-  curl -sS "${AUTH[@]}" "${BASE}/api/pipeline/status"
+  curl -sS --compressed "${AUTH[@]}" -H "Accept: application/json" \
+    "${BASE}/api/pipeline/status"
 }
 
 fetch_jobs_list() {
-  curl -sS "${AUTH[@]}" "${BASE}/api/jobs?view=list"
+  curl -sS --compressed "${AUTH[@]}" -H "Accept: application/json" \
+    "${BASE}/api/jobs?view=list"
+}
+
+fetch_jobs_revision() {
+  curl -sS --compressed "${AUTH[@]}" -H "Accept: application/json" \
+    "${BASE}/api/jobs/revision"
+}
+
+# After a run, the jobs list can briefly lag; also catches flaky proxies.
+fetch_jobs_list_when_ready() {
+  local expected_discovered="$1"
+  local resp=""
+  local n=0
+  local attempt=0
+  while [[ $attempt -lt 25 ]]; do
+    resp="$(fetch_jobs_list)"
+    if echo "$resp" | jq -e '.ok == true' >/dev/null 2>&1; then
+      n="$(echo "$resp" | jq -r '((.data // {}) | .jobs // []) | length')"
+      if [[ "$expected_discovered" -eq 0 ]] || [[ "$n" -gt 0 ]]; then
+        echo "$resp"
+        return 0
+      fi
+    fi
+    attempt=$((attempt + 1))
+    sleep 2
+  done
+  echo "$resp"
 }
 
 build_job_lines_html() {
@@ -83,7 +111,7 @@ build_job_lines_html() {
           end
         end
       end;
-    (.data.jobs // []) as $all |
+    (((.data // {}) | .jobs) // []) as $all |
     if ($all | length) == 0 then
       {total: 0, lines: [], usedFallback: false}
     else
@@ -136,8 +164,8 @@ if echo "$body" | jq -e '.data.isRunning == true' >/dev/null 2>&1; then
   exit 0
 fi
 
-resp="$(curl -sS "${AUTH[@]}" -X POST "${BASE}/api/pipeline/run" \
-  -H "Content-Type: application/json" -d '{}')"
+resp="$(curl -sS --compressed "${AUTH[@]}" -X POST "${BASE}/api/pipeline/run" \
+  -H "Accept: application/json" -H "Content-Type: application/json" -d '{}')"
 if ! echo "$resp" | jq -e '.ok == true' >/dev/null 2>&1; then
   _fail_json="$(echo "$resp" | jq -c . 2>/dev/null || echo "$resp")"
   send_tg_html "Jobber: POST /api/pipeline/run failed: $(tg_html_escape "$_fail_json")"
@@ -168,13 +196,16 @@ for _ in $(seq 1 720); do
     msg+=$'\n'"Discovered: ${disc}, processed: ${proc}."
     [[ -n "$err" ]] && msg+=$'\n'"<b>Error:</b> $(tg_html_escape "$err")"
 
-    jobs_resp="$(fetch_jobs_list)"
+    jobs_resp="$(fetch_jobs_list_when_ready "$disc")"
     if echo "$jobs_resp" | jq -e '.ok == true' >/dev/null 2>&1; then
-      sel="$(build_job_lines_html "$jobs_resp" "$started" "$completed" "$MAX_JOBS")" ||
-        sel='{"total":0,"lines":[]}'
+      list_n="$(echo "$jobs_resp" | jq -r '((.data // {}) | .jobs // []) | length')"
+      if ! sel="$(build_job_lines_html "$jobs_resp" "$started" "$completed" "$MAX_JOBS")"; then
+        sel='{"total":0,"lines":[],"usedFallback":false,"jqError":true}'
+      fi
       total="$(echo "$sel" | jq -r '.total // 0')"
       shown="$(echo "$sel" | jq -r '.lines | length')"
       used_fb="$(echo "$sel" | jq -r '.usedFallback // false')"
+      jq_err="$(echo "$sel" | jq -r '.jqError // false')"
       if [[ "$total" -gt 0 ]]; then
         if [[ "$used_fb" == "true" ]]; then
           msg+=$'\n\n'"<b>Recent jobs</b> (showing ${shown} of ${total}; time window did not match — links may include older discoveries):"
@@ -187,7 +218,29 @@ for _ in $(seq 1 720); do
           msg+=$'\n\n'"<i>…and ${rest} more not shown.</i>"
         fi
       else
-        msg+=$'\n\n'"<i>No job rows in the API list. Open the app for details.</i>"
+        rev_json="$(fetch_jobs_revision)"
+        rev_ok="$(echo "$rev_json" | jq -r 'if .ok == true then "1" else "0" end')"
+        rev_total="-1"
+        if [[ "$rev_ok" == "1" ]]; then
+          rev_total="$(echo "$rev_json" | jq -r '(.data.total // 0)')"
+        fi
+        msg+=$'\n\n'"<i>No job lines to show (list payload: ${list_n} rows)."
+        if [[ "$jq_err" == "true" ]]; then
+          msg+=" JSON/jq error while filtering.</i>"
+        else
+          msg+="</i>"
+        fi
+        if [[ "$rev_ok" == "1" ]]; then
+          msg+=$'\n'"<i>GET /api/jobs/revision reports <b>${rev_total}</b> jobs in DB.</i>"
+          if [[ "$rev_total" -gt 0 && "$list_n" -eq 0 ]]; then
+            msg+=$'\n'"<i>List response empty but DB has jobs — check reverse-proxy body limits, or multiple instances with different data dirs.</i>"
+          elif [[ "$rev_total" -eq 0 && "$disc" -gt 0 ]]; then
+            msg+=$'\n'"<i>Pipeline run reported ${disc} discovered but DB job count is 0 — wrong <code>JOBOPS_URL</code> (different server), or DB reset since the run.</i>"
+          fi
+        else
+          msg+=$'\n'"<i>Could not read /api/jobs/revision for diagnostics.</i>"
+        fi
+        msg+=$'\n'"<i>Open the app: $(tg_html_escape "${BASE}")</i>"
       fi
     else
       msg+=$'\n\n'"<i>Could not load GET /api/jobs for links.</i>"