ilia
1ce803fc9c
Web UI for channel/message cherry-pick, attachment preview, hydrate script, and workspace skip presets.
125 lines
4.0 KiB
Bash
Executable File
125 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download Slack export attachments using token= URLs embedded in the official export.
|
|
# Official exports include xoxe-... tokens on file URLs; slackdump hydrate rejects them.
|
|
#
|
|
# Usage:
|
|
# ./scripts/slack-hydrate-export.sh
|
|
# SLACK_EXPORT_DIR=... SLACK_HYDRATED_DIR=... ./scripts/slack-hydrate-export.sh
|
|
|
|
set -euo pipefail
|
|
|
|
SLACK_EXPORT_DIR="${SLACK_EXPORT_DIR:-$HOME/Downloads/Levkin Slack export Sep 29 2016 - May 27 2026}"
|
|
SLACK_HYDRATED_DIR="${SLACK_HYDRATED_DIR:-$HOME/Downloads/Levkin Slack export enriched}"
|
|
JOBS="${SLACK_HYDRATE_JOBS:-8}"
|
|
|
|
if [[ ! -d "$SLACK_EXPORT_DIR" ]]; then
|
|
echo "Export folder not found: $SLACK_EXPORT_DIR" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "==> Copying export JSON to $SLACK_HYDRATED_DIR (rsync, no deletes)..."
|
|
mkdir -p "$SLACK_HYDRATED_DIR"
|
|
rsync -a \
|
|
--exclude "__uploads" \
|
|
--exclude ".DS_Store" \
|
|
"$SLACK_EXPORT_DIR/" "$SLACK_HYDRATED_DIR/"
|
|
|
|
echo "==> Downloading attachments (parallel jobs=$JOBS)..."
|
|
python3 - "$SLACK_EXPORT_DIR" "$SLACK_HYDRATED_DIR" "$JOBS" <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
export_dir = Path(sys.argv[1])
|
|
hydrated_dir = Path(sys.argv[2])
|
|
jobs = int(sys.argv[3])
|
|
|
|
SKIP_ROOT = {
|
|
"channels.json",
|
|
"users.json",
|
|
"dms.json",
|
|
"groups.json",
|
|
"mpims.json",
|
|
"canvases.json",
|
|
"lists.json",
|
|
"huddle_transcripts.json",
|
|
"integration_logs.json",
|
|
"file_conversations.json",
|
|
}
|
|
|
|
|
|
def iter_files():
|
|
for path in export_dir.rglob("*.json"):
|
|
if path.name in SKIP_ROOT:
|
|
continue
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError):
|
|
continue
|
|
if not isinstance(data, list):
|
|
continue
|
|
for msg in data:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
for f in msg.get("files") or []:
|
|
if not isinstance(f, dict):
|
|
continue
|
|
fid = f.get("id")
|
|
name = f.get("name") or "file"
|
|
url = f.get("url_private_download") or f.get("url_private")
|
|
if not fid or not url:
|
|
continue
|
|
yield fid, name, url.replace("\\/", "/")
|
|
|
|
|
|
def download(fid: str, name: str, url: str) -> tuple[str, str]:
|
|
dest_dir = hydrated_dir / "__uploads" / fid
|
|
dest = dest_dir / name
|
|
if dest.exists() and dest.stat().st_size > 0:
|
|
return ("skip", str(dest))
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
req = urllib.request.Request(url, headers={"User-Agent": "slack-sieve-hydrate/1.0"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=120) as resp:
|
|
data = resp.read()
|
|
except urllib.error.HTTPError as exc:
|
|
return ("fail", f"{fid}/{name}: HTTP {exc.code}")
|
|
except Exception as exc: # noqa: BLE001
|
|
return ("fail", f"{fid}/{name}: {exc}")
|
|
dest.write_bytes(data)
|
|
return ("ok", str(dest))
|
|
|
|
|
|
tasks = list({(fid, name, url) for fid, name, url in iter_files()})
|
|
print(f"Found {len(tasks)} unique attachments")
|
|
|
|
ok = skip = fail = 0
|
|
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
futures = {pool.submit(download, *t): t for t in tasks}
|
|
for i, fut in enumerate(as_completed(futures), 1):
|
|
status, detail = fut.result()
|
|
if status == "ok":
|
|
ok += 1
|
|
elif status == "skip":
|
|
skip += 1
|
|
else:
|
|
fail += 1
|
|
print(detail, file=sys.stderr)
|
|
if i % 100 == 0 or i == len(tasks):
|
|
print(f" progress {i}/{len(tasks)} ok={ok} skip={skip} fail={fail}")
|
|
|
|
print(f"Done: ok={ok} skip={skip} fail={fail}")
|
|
print(f"Hydrated export: {hydrated_dir}")
|
|
print(f"Review: SLACK_EXPORT_PATH='{hydrated_dir}' make review")
|
|
if fail and ok == 0:
|
|
sys.exit(1)
|
|
if fail:
|
|
print(f"Warning: {fail} files failed (often expired export token on old uploads).", file=sys.stderr)
|
|
PY
|