diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..5928221 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,12 @@ +# Slack export fixtures historically embedded expired xoxe URL tokens. +# Working tree is redacted to xoxe-REDACTED; allowlist covers pre-redaction commits. +title = "slack-sieve" + +[allowlist] +description = "Test fixtures (tokens redacted going forward)" +paths = [ + '''tests/fixtures/.*''', +] +regexes = [ + '''xoxe-REDACTED''', +] diff --git a/scripts/build-test-fixture.py b/scripts/build-test-fixture.py index d5ccdb8..9285e02 100644 --- a/scripts/build-test-fixture.py +++ b/scripts/build-test-fixture.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import re import shutil import sys from pathlib import Path @@ -13,6 +14,13 @@ DEFAULT_SRC = Path.home() / "Downloads/Levkin Slack export enriched" DST = REPO / "tests" / "fixtures" / "mini-export" CHANNELS = ("medical", "zoey", "153niagara") MAX_DAY_FILES = 4 +SLACK_TOKEN_RE = re.compile( + r"xoxe-[0-9A-Za-z.-]+|xoxp-[0-9A-Za-z-]+|xoxb-[0-9A-Za-z-]+|xoxc-[0-9A-Za-z-]+" +) + + +def redact_tokens(text: str) -> str: + return SLACK_TOKEN_RE.sub("xoxe-REDACTED", text) def main() -> int: @@ -44,8 +52,9 @@ def main() -> int: continue ddir.mkdir() for jf in sorted(sdir.glob("*.json"))[:MAX_DAY_FILES]: - shutil.copy2(jf, ddir / jf.name) - data = json.loads(jf.read_text(encoding="utf-8")) + raw = jf.read_text(encoding="utf-8") + (ddir / jf.name).write_text(redact_tokens(raw), encoding="utf-8") + data = json.loads(raw) if isinstance(data, list): total_msgs += len(data) diff --git a/tests/fixtures/mini-export/153niagara/2016-10-11.json b/tests/fixtures/mini-export/153niagara/2016-10-11.json index 0cb7f73..178e852 100644 --- a/tests/fixtures/mini-export/153niagara/2016-10-11.json +++ b/tests/fixtures/mini-export/153niagara/2016-10-11.json @@ -23,8 +23,8 @@ "name": "Proof_read", "title": "Proof read", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", @@ -110,8 +110,8 @@ "name": "Proof_read", "title": "Proof read", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", @@ -153,8 +153,8 @@ "name": "Proof_read", "title": "Proof read", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", diff --git a/tests/fixtures/mini-export/153niagara/2016-10-18.json b/tests/fixtures/mini-export/153niagara/2016-10-18.json index 598ce01..1b8d3ce 100644 --- a/tests/fixtures/mini-export/153niagara/2016-10-18.json +++ b/tests/fixtures/mini-export/153niagara/2016-10-18.json @@ -23,8 +23,8 @@ "name": "draft_-_hot_plate", "title": "draft - hot plate", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", @@ -221,8 +221,8 @@ "name": "draft_-_hot_plate", "title": "draft - hot plate", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", @@ -264,8 +264,8 @@ "name": "draft_-_hot_plate", "title": "draft - hot plate", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", diff --git a/tests/fixtures/mini-export/medical/2018-09-13.json b/tests/fixtures/mini-export/medical/2018-09-13.json index 2510f0a..b23551f 100644 --- a/tests/fixtures/mini-export/medical/2018-09-13.json +++ b/tests/fixtures/mini-export/medical/2018-09-13.json @@ -23,10 +23,10 @@ "name": "img20180913_09120499.pdf", "title": "img20180913_09120499.pdf", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/img20180913_09120499.pdf?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/download\/img20180913_09120499.pdf?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/img20180913_09120499.pdf?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/download\/img20180913_09120499.pdf?token=xoxe-REDACTED", "media_display_type": "unknown", - "thumb_pdf": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-FCT24D5B7-45e94cde7b\/img20180913_09120499_thumb_pdf.png?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_pdf": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-FCT24D5B7-45e94cde7b\/img20180913_09120499_thumb_pdf.png?t=xoxe-REDACTED", "thumb_pdf_w": 930, "thumb_pdf_h": 1196, "permalink": "https:\/\/levkine.slack.com\/files\/U2HHNN1HQ\/FCT24D5B7\/img20180913_09120499.pdf", diff --git a/tests/fixtures/mini-export/zoey/2020-04-18.json b/tests/fixtures/mini-export/zoey/2020-04-18.json index 1f1cb0e..9e8deec 100644 --- a/tests/fixtures/mini-export/zoey/2020-04-18.json +++ b/tests/fixtures/mini-export/zoey/2020-04-18.json @@ -76,28 +76,28 @@ "name": "IMG_20200418_090219.jpg", "title": "IMG_20200418_090219.jpg", "is_modified_by_ai": false, - "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/img_20200418_090219.jpg?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/download\/img_20200418_090219.jpg?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/img_20200418_090219.jpg?token=xoxe-REDACTED", + "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/download\/img_20200418_090219.jpg?token=xoxe-REDACTED", "media_display_type": "unknown", - "thumb_64": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_64.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "thumb_80": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_80.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "thumb_360": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_360.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_64": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_64.jpg?t=xoxe-REDACTED", + "thumb_80": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_80.jpg?t=xoxe-REDACTED", + "thumb_360": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_360.jpg?t=xoxe-REDACTED", "thumb_360_w": 360, "thumb_360_h": 270, - "thumb_480": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_480.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_480": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_480.jpg?t=xoxe-REDACTED", "thumb_480_w": 480, "thumb_480_h": 360, - "thumb_160": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_160.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", - "thumb_720": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_720.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_160": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_160.jpg?t=xoxe-REDACTED", + "thumb_720": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_720.jpg?t=xoxe-REDACTED", "thumb_720_w": 720, "thumb_720_h": 540, - "thumb_800": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_800.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_800": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_800.jpg?t=xoxe-REDACTED", "thumb_800_w": 800, "thumb_800_h": 600, - "thumb_960": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_960.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_960": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_960.jpg?t=xoxe-REDACTED", "thumb_960_w": 960, "thumb_960_h": 720, - "thumb_1024": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_1024.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", + "thumb_1024": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_1024.jpg?t=xoxe-REDACTED", "thumb_1024_w": 1024, "thumb_1024_h": 768, "original_w": 4032, diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index 70908d2..9667d4b 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -189,7 +189,7 @@ def test_fetch_http_error(rich_export, monkeypatch): "name": "b.bin", "local_relpath": "__uploads/X2/b.bin", "local_exists": False, - "url_private_download": "https://example.com/x?token=xoxe-abc", + "url_private_download": "https://example.com/x?token=xoxe-REDACTED", } } diff --git a/tests/test_slack_export_extended.py b/tests/test_slack_export_extended.py index edd73ac..bfabd87 100644 --- a/tests/test_slack_export_extended.py +++ b/tests/test_slack_export_extended.py @@ -190,7 +190,7 @@ def test_fetch_file_bytes_remote(export_with_synthetic, monkeypatch): monkeypatch.setattr("urllib.request.urlopen", lambda *a, **k: _Resp()) rec_entry = export_with_synthetic.ensure_file_index()["F1"] - rec_entry["url_private_download"] = "https://files.slack.com/x?token=xoxe-test" + rec_entry["url_private_download"] = "https://files.slack.com/x?token=xoxe-REDACTED" data, meta = export_with_synthetic.fetch_file_bytes("F1") assert data == b"remote-bytes"