slack-sieve/scripts/prepare-export.py

#!/usr/bin/env python3
"""Build a filtered Slack-shaped export from manifest.json for mmetl/mmctl."""

from __future__ import annotations

import json
import shutil
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[1]
MANIFEST_PATH = REPO_ROOT / "manifest.json"
DEFAULT_EXPORT = Path.home() / "Downloads/Levkin Slack export enriched"
DEFAULT_OUT = REPO_ROOT / "prepared-export"


def message_key(channel: str, ts: str | float) -> str:
    return f"{channel}:{ts}"


def channel_included(entry: dict) -> bool:
    status = entry.get("status")
    if status == "skip":
        return False
    if status == "import" and entry.get("import_mode", "all") != "pick":
        return True
    if status in ("import", "partial"):
        picks = entry.get("messages") or {}
        return any(
            (sel.get("action") in ("import", "merge") or sel.get("checked"))
            for sel in picks.values()
            if isinstance(sel, dict)
        )
    return False


def message_included(entry: dict, key: str) -> bool:
    status = entry.get("status")
    if status == "skip":
        return False
    if status == "import" and entry.get("import_mode", "all") != "pick":
        return True
    picks = entry.get("messages") or {}
    sel = picks.get(key)
    if not sel:
        return False
    if sel.get("action") == "skip":
        return False
    return sel.get("action") in ("import", "merge") or bool(sel.get("checked"))


def filter_files_on_message(entry: dict, key: str, raw: dict) -> dict:
    """Return a copy of the Slack message with only manifest-selected files."""
    picks = entry.get("messages") or {}
    sel = picks.get(key) or {}
    file_flags = sel.get("files") if isinstance(sel.get("files"), dict) else {}
    files = raw.get("files")
    if not files:
        return raw
    out = dict(raw)
    kept = []
    for f in files:
        if not isinstance(f, dict):
            continue
        fid = f.get("id") or f.get("name")
        if not fid:
            continue
        if file_flags and file_flags.get(fid) is False:
            continue
        kept.append(f)
    if not kept:
        out.pop("files", None)
    else:
        out["files"] = kept
    return out


def prepare_export(
    export_root: Path,
    manifest: dict,
    out_root: Path,
    *,
    export_mod=None,
) -> dict[str, int]:
    """
    Write filtered export to out_root. Returns per-channel message counts.
    """
    sys.path.insert(0, str(REPO_ROOT))
    from slack_export import SlackExport, load_json

    export = export_mod or SlackExport(export_root)
    out_root = out_root.resolve()
    if out_root.exists():
        shutil.rmtree(out_root)
    out_root.mkdir(parents=True)

    channels_meta = load_json(export.paths.channels_json)
    if not isinstance(channels_meta, list):
        channels_meta = []

    manifest_channels = manifest.get("channels") or {}
    included_names: set[str] = set()
    stats: dict[str, int] = {}

    for ch in channels_meta:
        name = ch.get("name")
        if not name:
            continue
        entry = manifest_channels.get(name) or {}
        if not channel_included(entry):
            continue
        included_names.add(name)

    # channels.json subset
    filtered_meta = [c for c in channels_meta if c.get("name") in included_names]
    (out_root / "channels.json").write_text(
        json.dumps(filtered_meta, indent=2, ensure_ascii=False) + "\n",
        encoding="utf-8",
    )

    # users.json — copy whole (mmetl needs user mapping)
    users_src = export.paths.users_json
    if users_src.is_file():
        shutil.copy2(users_src, out_root / "users.json")

    uploads_out = out_root / "__uploads"
    uploads_src = export.paths.root / "__uploads"

    for name in sorted(included_names):
        entry = manifest_channels[name]
        ch_out = out_root / name
        ch_out.mkdir(parents=True)
        count = 0
        for date in export.list_channel_date_files(name):
            raw_list = export.read_messages_file(name, date)
            if not isinstance(raw_list, list):
                continue
            kept: list[dict] = []
            for raw in raw_list:
                if not isinstance(raw, dict):
                    continue
                ts = raw.get("ts")
                if ts is None:
                    continue
                key = message_key(name, ts)
                if not message_included(entry, key):
                    continue
                msg = filter_files_on_message(entry, key, raw)
                kept.append(msg)
                count += 1
                for f in msg.get("files") or []:
                    if not isinstance(f, dict):
                        continue
                    fid = f.get("id")
                    fname = f.get("name")
                    if fid and fname and uploads_src.is_dir():
                        src = uploads_src / fid / fname
                        if src.is_file():
                            dest = uploads_out / fid / fname
                            dest.parent.mkdir(parents=True, exist_ok=True)
                            shutil.copy2(src, dest)
            if kept:
                (ch_out / f"{date}.json").write_text(
                    json.dumps(kept, indent=2, ensure_ascii=False) + "\n",
                    encoding="utf-8",
                )
        stats[name] = count

    summary = {
        "channels": len(stats),
        "messages": sum(stats.values()),
        "per_channel": stats,
    }
    (out_root / "prepare-export-summary.json").write_text(
        json.dumps(summary, indent=2) + "\n",
        encoding="utf-8",
    )
    return stats


def main() -> int:
    export_path = DEFAULT_EXPORT
    out_path = DEFAULT_OUT
    manifest_path = MANIFEST_PATH

    args = sys.argv[1:]
    i = 0
    while i < len(args):
        if args[i] == "--export" and i + 1 < len(args):
            export_path = Path(args[i + 1]).expanduser()
            i += 2
        elif args[i] == "--out" and i + 1 < len(args):
            out_path = Path(args[i + 1]).expanduser()
            i += 2
        elif args[i] == "--manifest" and i + 1 < len(args):
            manifest_path = Path(args[i + 1]).expanduser()
            i += 2
        elif not args[i].startswith("-"):
            export_path = Path(args[i]).expanduser()
            i += 1
        else:
            i += 1

    export_path = export_path.resolve()
    if not export_path.is_dir():
        print(f"Export not found: {export_path}", file=sys.stderr)
        return 1
    if not manifest_path.is_file():
        print(f"Manifest not found: {manifest_path}", file=sys.stderr)
        return 1

    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    stats = prepare_export(export_path, manifest, out_path.resolve())
    total = sum(stats.values())
    print(f"Wrote {out_path}")
    print(f"  {len(stats)} channels, {total} messages")
    for name, n in sorted(stats.items()):
        print(f"    #{name}: {n}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())