#!/usr/bin/env bash
# Download Slack export attachments using token= URLs embedded in the official export.
# Official exports include xoxe-... tokens on file URLs; slackdump hydrate rejects them.
#
# Usage:
#   ./scripts/slack-hydrate-export.sh
#   SLACK_EXPORT_DIR=... SLACK_HYDRATED_DIR=... ./scripts/slack-hydrate-export.sh

set -euo pipefail

SLACK_EXPORT_DIR="${SLACK_EXPORT_DIR:-$HOME/Downloads/Levkin Slack export Sep 29 2016 - May 27 2026}"
SLACK_HYDRATED_DIR="${SLACK_HYDRATED_DIR:-$HOME/Downloads/Levkin Slack export enriched}"
JOBS="${SLACK_HYDRATE_JOBS:-8}"

if [[ ! -d "$SLACK_EXPORT_DIR" ]]; then
  echo "Export folder not found: $SLACK_EXPORT_DIR" >&2
  exit 1
fi

echo "==> Copying export JSON to $SLACK_HYDRATED_DIR (rsync, no deletes)..."
mkdir -p "$SLACK_HYDRATED_DIR"
rsync -a \
  --exclude "__uploads" \
  --exclude ".DS_Store" \
  "$SLACK_EXPORT_DIR/" "$SLACK_HYDRATED_DIR/"

echo "==> Downloading attachments (parallel jobs=$JOBS)..."
python3 - "$SLACK_EXPORT_DIR" "$SLACK_HYDRATED_DIR" "$JOBS" <<'PY'
from __future__ import annotations

import json
import os
import sys
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

export_dir = Path(sys.argv[1])
hydrated_dir = Path(sys.argv[2])
jobs = int(sys.argv[3])

SKIP_ROOT = {
    "channels.json",
    "users.json",
    "dms.json",
    "groups.json",
    "mpims.json",
    "canvases.json",
    "lists.json",
    "huddle_transcripts.json",
    "integration_logs.json",
    "file_conversations.json",
}


def iter_files():
    for path in export_dir.rglob("*.json"):
        if path.name in SKIP_ROOT:
            continue
        try:
            data = json.loads(path.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            continue
        if not isinstance(data, list):
            continue
        for msg in data:
            if not isinstance(msg, dict):
                continue
            for f in msg.get("files") or []:
                if not isinstance(f, dict):
                    continue
                fid = f.get("id")
                name = f.get("name") or "file"
                url = f.get("url_private_download") or f.get("url_private")
                if not fid or not url:
                    continue
                yield fid, name, url.replace("\\/", "/")


def download(fid: str, name: str, url: str) -> tuple[str, str]:
    dest_dir = hydrated_dir / "__uploads" / fid
    dest = dest_dir / name
    if dest.exists() and dest.stat().st_size > 0:
        return ("skip", str(dest))
    dest_dir.mkdir(parents=True, exist_ok=True)
    req = urllib.request.Request(url, headers={"User-Agent": "slack-sieve-hydrate/1.0"})
    try:
        with urllib.request.urlopen(req, timeout=120) as resp:
            data = resp.read()
    except urllib.error.HTTPError as exc:
        return ("fail", f"{fid}/{name}: HTTP {exc.code}")
    except Exception as exc:  # noqa: BLE001
        return ("fail", f"{fid}/{name}: {exc}")
    dest.write_bytes(data)
    return ("ok", str(dest))


tasks = list({(fid, name, url) for fid, name, url in iter_files()})
print(f"Found {len(tasks)} unique attachments")

ok = skip = fail = 0
with ThreadPoolExecutor(max_workers=jobs) as pool:
    futures = {pool.submit(download, *t): t for t in tasks}
    for i, fut in enumerate(as_completed(futures), 1):
        status, detail = fut.result()
        if status == "ok":
            ok += 1
        elif status == "skip":
            skip += 1
        else:
            fail += 1
            print(detail, file=sys.stderr)
        if i % 100 == 0 or i == len(tasks):
            print(f"  progress {i}/{len(tasks)} ok={ok} skip={skip} fail={fail}")

print(f"Done: ok={ok} skip={skip} fail={fail}")
print(f"Hydrated export: {hydrated_dir}")
print(f"Review: SLACK_EXPORT_PATH='{hydrated_dir}' make review")
if fail and ok == 0:
    sys.exit(1)
if fail:
    print(f"Warning: {fail} files failed (often expired export token on old uploads).", file=sys.stderr)
PY