Initial import: Slack export triage UI for Mattermost migration.

Web UI for channel/message cherry-pick, attachment preview, hydrate script, and workspace skip presets.
2026-05-29 13:53:28 -04:00
commit 1ce803fc9c
12 changed files with 3781 additions and 0 deletions
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Download Slack export attachments using token= URLs embedded in the official export.
+# Official exports include xoxe-... tokens on file URLs; slackdump hydrate rejects them.
+#
+# Usage:
+#   ./scripts/slack-hydrate-export.sh
+#   SLACK_EXPORT_DIR=... SLACK_HYDRATED_DIR=... ./scripts/slack-hydrate-export.sh
+
+set -euo pipefail
+
+SLACK_EXPORT_DIR="${SLACK_EXPORT_DIR:-$HOME/Downloads/Levkin Slack export Sep 29 2016 - May 27 2026}"
+SLACK_HYDRATED_DIR="${SLACK_HYDRATED_DIR:-$HOME/Downloads/Levkin Slack export enriched}"
+JOBS="${SLACK_HYDRATE_JOBS:-8}"
+
+if [[ ! -d "$SLACK_EXPORT_DIR" ]]; then
+  echo "Export folder not found: $SLACK_EXPORT_DIR" >&2
+  exit 1
+fi
+
+echo "==> Copying export JSON to $SLACK_HYDRATED_DIR (rsync, no deletes)..."
+mkdir -p "$SLACK_HYDRATED_DIR"
+rsync -a \
+  --exclude "__uploads" \
+  --exclude ".DS_Store" \
+  "$SLACK_EXPORT_DIR/" "$SLACK_HYDRATED_DIR/"
+
+echo "==> Downloading attachments (parallel jobs=$JOBS)..."
+python3 - "$SLACK_EXPORT_DIR" "$SLACK_HYDRATED_DIR" "$JOBS" <<'PY'
+from __future__ import annotations
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+export_dir = Path(sys.argv[1])
+hydrated_dir = Path(sys.argv[2])
+jobs = int(sys.argv[3])
+
+SKIP_ROOT = {
+    "channels.json",
+    "users.json",
+    "dms.json",
+    "groups.json",
+    "mpims.json",
+    "canvases.json",
+    "lists.json",
+    "huddle_transcripts.json",
+    "integration_logs.json",
+    "file_conversations.json",
+}
+
+
+def iter_files():
+    for path in export_dir.rglob("*.json"):
+        if path.name in SKIP_ROOT:
+            continue
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+        if not isinstance(data, list):
+            continue
+        for msg in data:
+            if not isinstance(msg, dict):
+                continue
+            for f in msg.get("files") or []:
+                if not isinstance(f, dict):
+                    continue
+                fid = f.get("id")
+                name = f.get("name") or "file"
+                url = f.get("url_private_download") or f.get("url_private")
+                if not fid or not url:
+                    continue
+                yield fid, name, url.replace("\\/", "/")
+
+
+def download(fid: str, name: str, url: str) -> tuple[str, str]:
+    dest_dir = hydrated_dir / "__uploads" / fid
+    dest = dest_dir / name
+    if dest.exists() and dest.stat().st_size > 0:
+        return ("skip", str(dest))
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    req = urllib.request.Request(url, headers={"User-Agent": "slack-sieve-hydrate/1.0"})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            data = resp.read()
+    except urllib.error.HTTPError as exc:
+        return ("fail", f"{fid}/{name}: HTTP {exc.code}")
+    except Exception as exc:  # noqa: BLE001
+        return ("fail", f"{fid}/{name}: {exc}")
+    dest.write_bytes(data)
+    return ("ok", str(dest))
+
+
+tasks = list({(fid, name, url) for fid, name, url in iter_files()})
+print(f"Found {len(tasks)} unique attachments")
+
+ok = skip = fail = 0
+with ThreadPoolExecutor(max_workers=jobs) as pool:
+    futures = {pool.submit(download, *t): t for t in tasks}
+    for i, fut in enumerate(as_completed(futures), 1):
+        status, detail = fut.result()
+        if status == "ok":
+            ok += 1
+        elif status == "skip":
+            skip += 1
+        else:
+            fail += 1
+            print(detail, file=sys.stderr)
+        if i % 100 == 0 or i == len(tasks):
+            print(f"  progress {i}/{len(tasks)} ok={ok} skip={skip} fail={fail}")
+
+print(f"Done: ok={ok} skip={skip} fail={fail}")
+print(f"Hydrated export: {hydrated_dir}")
+print(f"Review: SLACK_EXPORT_PATH='{hydrated_dir}' make review")
+if fail and ok == 0:
+    sys.exit(1)
+if fail:
+    print(f"Warning: {fail} files failed (often expired export token on old uploads).", file=sys.stderr)
+PY