ilia/slack-sieve
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Redact Slack xoxe tokens from fixtures and allowlist for gitleaks.
CI / skip-ci-check (pull_request) Successful in 7s
Details
CI / secret-scan (pull_request) Successful in 6s
Details
CI / python-ci (pull_request) Successful in 21s
Details

Browse Source 
Export URL tokens in mini-export fixtures tripped secret-scan; replace with placeholders, stop regenerating them, and allowlist fixture paths for pre-redaction history.
This commit is contained in:
ilia
2026-07-10 13:45:02 -04:00
parent 934b674995
commit 90b632694e
8 changed files with 51 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitleaks.toml
+12
View File
@@ -0,0 +1,12 @@
# Slack export fixtures historically embedded expired xoxe URL tokens.
# Working tree is redacted to xoxe-REDACTED; allowlist covers pre-redaction commits.
title = "slack-sieve"
[allowlist]
description = "Test fixtures (tokens redacted going forward)"
paths = [
'''tests/fixtures/.*''',
]
regexes = [
'''xoxe-REDACTED''',
]
scripts/build-test-fixture.py
+11 -2
View File
@@ -4,6 +4,7 @@
from __future__ import annotations from __future__ import annotations
import json import json
import re
import shutil import shutil
import sys import sys
from pathlib import Path from pathlib import Path
@@ -13,6 +14,13 @@ DEFAULT_SRC = Path.home() / "Downloads/Levkin Slack export enriched"
DST = REPO / "tests" / "fixtures" / "mini-export" DST = REPO / "tests" / "fixtures" / "mini-export"
CHANNELS = ("medical", "zoey", "153niagara") CHANNELS = ("medical", "zoey", "153niagara")
MAX_DAY_FILES = 4 MAX_DAY_FILES = 4
SLACK_TOKEN_RE = re.compile(
r"xoxe-[0-9A-Za-z.-]+|xoxp-[0-9A-Za-z-]+|xoxb-[0-9A-Za-z-]+|xoxc-[0-9A-Za-z-]+"
)
def redact_tokens(text: str) -> str:
return SLACK_TOKEN_RE.sub("xoxe-REDACTED", text)
def main() -> int: def main() -> int:
@@ -44,8 +52,9 @@ def main() -> int:
continue continue
ddir.mkdir() ddir.mkdir()
for jf in sorted(sdir.glob("*.json"))[:MAX_DAY_FILES]: for jf in sorted(sdir.glob("*.json"))[:MAX_DAY_FILES]:
shutil.copy2(jf, ddir / jf.name) raw = jf.read_text(encoding="utf-8")
data = json.loads(jf.read_text(encoding="utf-8")) (ddir / jf.name).write_text(redact_tokens(raw), encoding="utf-8")
data = json.loads(raw)
if isinstance(data, list): if isinstance(data, list):
total_msgs += len(data) total_msgs += len(data)
tests/fixtures/mini-export/153niagara/2016-10-11.json
Vendored
+6 -6
View File
@@ -23,8 +23,8 @@
"name": "Proof_read", "name": "Proof_read",
"title": "Proof read", "title": "Proof read",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile",
@@ -110,8 +110,8 @@
"name": "Proof_read", "name": "Proof_read",
"title": "Proof read", "title": "Proof read",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile",
@@ -153,8 +153,8 @@
"name": "Proof_read", "name": "Proof_read",
"title": "Proof read", "title": "Proof read",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049MQBLJBF\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049MQBLJBF-8ae09c2c9a",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049MQBLJBF\/mobile",
tests/fixtures/mini-export/153niagara/2016-10-18.json
Vendored
+6 -6
View File
@@ -23,8 +23,8 @@
"name": "draft_-_hot_plate", "name": "draft_-_hot_plate",
"title": "draft - hot plate", "title": "draft - hot plate",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile",
@@ -221,8 +221,8 @@
"name": "draft_-_hot_plate", "name": "draft_-_hot_plate",
"title": "draft - hot plate", "title": "draft - hot plate",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile",
@@ -264,8 +264,8 @@
"name": "draft_-_hot_plate", "name": "draft_-_hot_plate",
"title": "draft - hot plate", "title": "draft - hot plate",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/canvas?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F049QLYJZSN\/download\/canvas?token=xoxe-REDACTED",
"permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN", "permalink": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN",
"permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162", "permalink_public": "https:\/\/slack-files.com\/T2HHNN1FU-F049QLYJZSN-1ded605162",
"url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile", "url_static_preview": "https:\/\/levkine.slack.com\/docs\/T2HHNN1FU\/F049QLYJZSN\/mobile",
tests/fixtures/mini-export/medical/2018-09-13.json
Vendored
+3 -3
View File
@@ -23,10 +23,10 @@
"name": "img20180913_09120499.pdf", "name": "img20180913_09120499.pdf",
"title": "img20180913_09120499.pdf", "title": "img20180913_09120499.pdf",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/img20180913_09120499.pdf?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/img20180913_09120499.pdf?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/download\/img20180913_09120499.pdf?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-FCT24D5B7\/download\/img20180913_09120499.pdf?token=xoxe-REDACTED",
"media_display_type": "unknown", "media_display_type": "unknown",
"thumb_pdf": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-FCT24D5B7-45e94cde7b\/img20180913_09120499_thumb_pdf.png?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_pdf": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-FCT24D5B7-45e94cde7b\/img20180913_09120499_thumb_pdf.png?t=xoxe-REDACTED",
"thumb_pdf_w": 930, "thumb_pdf_w": 930,
"thumb_pdf_h": 1196, "thumb_pdf_h": 1196,
"permalink": "https:\/\/levkine.slack.com\/files\/U2HHNN1HQ\/FCT24D5B7\/img20180913_09120499.pdf", "permalink": "https:\/\/levkine.slack.com\/files\/U2HHNN1HQ\/FCT24D5B7\/img20180913_09120499.pdf",
tests/fixtures/mini-export/zoey/2020-04-18.json
Vendored
+11 -11
View File
@@ -76,28 +76,28 @@
"name": "IMG_20200418_090219.jpg", "name": "IMG_20200418_090219.jpg",
"title": "IMG_20200418_090219.jpg", "title": "IMG_20200418_090219.jpg",
"is_modified_by_ai": false, "is_modified_by_ai": false,
"url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/img_20200418_090219.jpg?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/img_20200418_090219.jpg?token=xoxe-REDACTED",
"url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/download\/img_20200418_090219.jpg?token=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "url_private_download": "https:\/\/files.slack.com\/files-pri\/T2HHNN1FU-F011NH76RUP\/download\/img_20200418_090219.jpg?token=xoxe-REDACTED",
"media_display_type": "unknown", "media_display_type": "unknown",
"thumb_64": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_64.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_64": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_64.jpg?t=xoxe-REDACTED",
"thumb_80": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_80.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_80": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_80.jpg?t=xoxe-REDACTED",
"thumb_360": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_360.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_360": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_360.jpg?t=xoxe-REDACTED",
"thumb_360_w": 360, "thumb_360_w": 360,
"thumb_360_h": 270, "thumb_360_h": 270,
"thumb_480": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_480.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_480": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_480.jpg?t=xoxe-REDACTED",
"thumb_480_w": 480, "thumb_480_w": 480,
"thumb_480_h": 360, "thumb_480_h": 360,
"thumb_160": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_160.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_160": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_160.jpg?t=xoxe-REDACTED",
"thumb_720": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_720.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_720": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_720.jpg?t=xoxe-REDACTED",
"thumb_720_w": 720, "thumb_720_w": 720,
"thumb_720_h": 540, "thumb_720_h": 540,
"thumb_800": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_800.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_800": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_800.jpg?t=xoxe-REDACTED",
"thumb_800_w": 800, "thumb_800_w": 800,
"thumb_800_h": 600, "thumb_800_h": 600,
"thumb_960": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_960.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_960": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_960.jpg?t=xoxe-REDACTED",
"thumb_960_w": 960, "thumb_960_w": 960,
"thumb_960_h": 720, "thumb_960_h": 720,
"thumb_1024": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_1024.jpg?t=xoxe-85600749538-11214969224067-11215699816469-13b604e355c6cb0f8b9d8a5c2dd39dd1", "thumb_1024": "https:\/\/files.slack.com\/files-tmb\/T2HHNN1FU-F011NH76RUP-165eb608df\/img_20200418_090219_1024.jpg?t=xoxe-REDACTED",
"thumb_1024_w": 1024, "thumb_1024_w": 1024,
"thumb_1024_h": 768, "thumb_1024_h": 768,
"original_w": 4032, "original_w": 4032,
tests/test_coverage_gaps.py
+1 -1
View File
@@ -189,7 +189,7 @@ def test_fetch_http_error(rich_export, monkeypatch):
"name": "b.bin", "name": "b.bin",
"local_relpath": "__uploads/X2/b.bin", "local_relpath": "__uploads/X2/b.bin",
"local_exists": False, "local_exists": False,
"url_private_download": "https://example.com/x?token=xoxe-abc", "url_private_download": "https://example.com/x?token=xoxe-REDACTED",
} }
} }
tests/test_slack_export_extended.py
+1 -1
View File
@@ -190,7 +190,7 @@ def test_fetch_file_bytes_remote(export_with_synthetic, monkeypatch):
monkeypatch.setattr("urllib.request.urlopen", lambda *a, **k: _Resp()) monkeypatch.setattr("urllib.request.urlopen", lambda *a, **k: _Resp())
rec_entry = export_with_synthetic.ensure_file_index()["F1"] rec_entry = export_with_synthetic.ensure_file_index()["F1"]
rec_entry["url_private_download"] = "https://files.slack.com/x?token=xoxe-test" rec_entry["url_private_download"] = "https://files.slack.com/x?token=xoxe-REDACTED"
data, meta = export_with_synthetic.fetch_file_bytes("F1") data, meta = export_with_synthetic.fetch_file_bytes("F1")
assert data == b"remote-bytes" assert data == b"remote-bytes"