Add triage tests, prepare-export, and UI fixes for Levkin migration.
CI / skip-ci-check (push) Successful in 4s
CI / secret-scan (push) Failing after 2s
CI / python-ci (push) Successful in 25s

Cherry-pick heuristics, manifest merge, global file search, scrollbar styling, and pytest/Playwright coverage unblock review and filtered export for mmetl.
This commit is contained in:
2026-06-03 15:42:48 -04:00
parent 88b7082f4f
commit 4b034d19f6
48 changed files with 6904 additions and 126 deletions
+326 -103
View File
@@ -33,6 +33,8 @@ const state = {
const els = {
stats: document.getElementById("stats"),
globalSearch: document.getElementById("global-search"),
globalSearchMode: document.getElementById("global-search-mode"),
globalFileType: document.getElementById("global-file-type"),
globalSearchResults: document.getElementById("global-search-results"),
sidebarSearch: document.getElementById("sidebar-search"),
filterStatus: document.getElementById("filter-status"),
@@ -43,6 +45,9 @@ const els = {
channelMeta: document.getElementById("channel-meta"),
importModePick: document.getElementById("import-mode-pick"),
messageSearch: document.getElementById("message-search"),
filterAttachmentType: document.getElementById("filter-attachment-type"),
btnSelectImages: document.getElementById("btn-select-images"),
btnSelectFiles: document.getElementById("btn-select-files"),
filterHideBots: document.getElementById("filter-hide-bots"),
filterHideSystem: document.getElementById("filter-hide-system"),
filterOnlySelected: document.getElementById("filter-only-selected"),
@@ -115,6 +120,22 @@ function isImageFile(file) {
return ["png", "jpg", "jpeg", "gif", "webp", "bmp", "svg", "heic"].includes(ext);
}
function fileKind(file) {
const mt = (file.mimetype || "").toLowerCase();
const name = (file.name || "").toLowerCase();
const ext = name.includes(".") ? name.split(".").pop() : "";
if (isImageFile(file)) return "image";
if (ext === "pdf" || mt === "application/pdf") return "pdf";
if (["doc", "docx"].includes(ext) || mt.includes("word")) return "doc";
if (["xls", "xlsx", "csv"].includes(ext) || mt.includes("spreadsheet") || mt.includes("excel"))
return "spreadsheet";
return "other";
}
function messageKinds(msg) {
return new Set((msg.files || []).map(fileKind));
}
function fileExtLabel(name) {
const ext = (name || "").split(".").pop()?.toUpperCase() || "";
if (!ext || ext.length > 6) return "FILE";
@@ -383,20 +404,45 @@ function userEntry(id) {
return state.userManifest[id];
}
function channelUsesCherryPick(ch) {
return ch.import_mode === "pick" || ch.status === "partial";
}
function effectiveMsgAction(channelName, msg) {
const ch = channelEntry(channelName);
const sel = ch.messages[msg.id];
const explicit = sel?.action;
if (explicit && explicit !== "inherit") return explicit;
if (explicit === "import" || explicit === "merge" || explicit === "skip") {
return explicit;
}
if (ch.import_mode === "pick") {
return sel?.import === true || sel?.checked ? "import" : "skip";
if (channelUsesCherryPick(ch)) {
return "skip";
}
if (ch.status === "import") return "import";
if (ch.status === "skip") return "skip";
return "skip";
}
function clearMsgSelection(channelName, msgId) {
const ch = channelEntry(channelName);
if (ch.messages?.[msgId]) {
delete ch.messages[msgId];
}
}
function setMsgImport(channelName, msg, on) {
if (!on) {
clearMsgSelection(channelName, msg.id);
return;
}
ensureCherryPick();
const sel = msgEntry(channelName, msg.id);
sel.action = "import";
sel.checked = true;
sel.merge_target = "";
}
function isMsgSelected(channelName, msg) {
const action = effectiveMsgAction(channelName, msg);
return action === "import" || action === "merge";
@@ -423,6 +469,22 @@ function reviewedCount() {
return state.channels.filter((c) => channelEntry(c.name).reviewed).length;
}
function channelPickCount(name) {
const msgs = channelEntry(name).messages || {};
return Object.values(msgs).filter(
(m) => m.action === "import" || m.action === "merge"
).length;
}
function channelHasPicks(name) {
const entry = channelEntry(name);
return (
entry.status === "partial" ||
entry.import_mode === "pick" ||
channelPickCount(name) > 0
);
}
function visibleChannelList() {
const q = els.sidebarSearch.value.trim().toLowerCase();
const statusFilter = els.filterStatus.value;
@@ -432,7 +494,9 @@ function visibleChannelList() {
const entry = channelEntry(ch.name);
const status = entry.status === "partial" ? "partial" : entry.status;
if (hideArchived && ch.is_archived) return false;
if (statusFilter !== "all" && status !== statusFilter) return false;
if (statusFilter === "partial") {
if (!channelHasPicks(ch.name)) return false;
} else if (statusFilter !== "all" && status !== statusFilter) return false;
if (!q) return true;
return (
ch.name.toLowerCase().includes(q) ||
@@ -499,8 +563,13 @@ function renderSidebar() {
btn.type = "button";
btn.className =
"list-item" + (state.selectedChannel === ch.name ? " active" : "");
const picks = channelPickCount(ch.name);
const mode =
entry.import_mode === "pick" ? " · cherry-pick" : "";
entry.import_mode === "pick" || picks > 0
? picks > 0
? ` · ${formatNumber(picks)} picked`
: " · cherry-pick"
: "";
btn.innerHTML = `
<span class="status-dot ${dotStatus}"></span>
<div class="body">
@@ -608,13 +677,19 @@ function ensureCherryPick() {
function setMessageAction(msg, action, { advance = false } = {}) {
if (!state.selectedChannel || !msg) return;
ensureCherryPick();
const sel = msgEntry(state.selectedChannel, msg.id);
sel.action = action;
sel.checked = action === "import" || action === "merge";
if (action !== "merge") sel.merge_target = "";
if (action === "import" || action === "merge") {
ensureCherryPick();
const sel = msgEntry(state.selectedChannel, msg.id);
sel.action = action;
sel.checked = true;
if (action !== "merge") sel.merge_target = "";
} else {
clearMsgSelection(state.selectedChannel, msg.id);
}
renderAllMessages();
renderSidebar();
updateTriageBar();
scheduleAutosave();
if (advance) {
@@ -632,8 +707,7 @@ function toggleFocusedImport() {
const msg = state.allMessages.find((m) => m.id === state.focusMessageId);
if (!msg || !state.selectedChannel) return;
const sel = msgEntry(state.selectedChannel, msg.id);
const currentlyOn =
sel.action === "import" || sel.action === "merge" || sel.checked;
const currentlyOn = isMsgSelected(state.selectedChannel, msg);
setMessageAction(msg, currentlyOn ? "skip" : "import", { advance: false });
}
@@ -716,6 +790,15 @@ function messagePassesFilters(msg) {
if (els.filterOnlySelected.checked && state.selectedChannel) {
if (!isMsgSelected(state.selectedChannel, msg)) return false;
}
const att = els.filterAttachmentType?.value || "all";
if (att !== "all") {
const kinds = messageKinds(msg);
if (att === "any") {
if (!msg.files?.length) return false;
} else if (!kinds.has(att)) {
return false;
}
}
return true;
}
@@ -723,7 +806,7 @@ function messageRowHtml(msg, q) {
const ch = state.selectedChannel;
const action = effectiveMsgAction(ch, msg);
const sel = msgEntry(ch, msg.id);
const checked = action === "import" || action === "merge" || sel.checked;
const checked = action === "import" || action === "merge";
const classes = ["message"];
if (msg.is_thread_reply) classes.push("thread-reply");
@@ -796,6 +879,7 @@ function renderAllMessages() {
els.filterCount.textContent = `${shown} / ${state.allMessages.length} shown`;
renderSelectionSummary();
updateMessageNavStatus();
updateBulkSelectButtons();
if (state.focusMessageId) {
const row = els.messages.querySelector(
@@ -808,20 +892,21 @@ function renderAllMessages() {
function openMessageDetail(msg) {
state.selectedMessageId = msg.id;
const ch = state.selectedChannel;
const sel = msgEntry(ch, msg.id);
const saved = channelEntry(ch).messages[msg.id];
const isOn = isMsgSelected(ch, msg);
els.messageDetail.classList.remove("hidden");
els.msgPreview.textContent = `${msg.user_name}: ${(msg.text || "[no text]").slice(0, 200)}`;
els.msgAction.value =
sel.action && sel.action !== "inherit"
? sel.action
: isMsgSelected(ch, msg)
saved?.action && saved.action !== "inherit"
? saved.action
: isOn
? "import"
: "inherit";
els.msgMergeTarget.value = sel.merge_target || "";
els.msgMergeTarget.value = saved?.merge_target || "";
els.msgMergeTarget.classList.toggle("hidden", els.msgAction.value !== "merge");
els.msgNotes.value = sel.notes || "";
els.msgNotes.value = saved?.notes || "";
if (msg.is_thread_parent || msg.thread_reply_count > 0) {
els.btnThreadAll.classList.remove("hidden");
@@ -846,7 +931,7 @@ function openMessageDetail(msg) {
els.fileToggles.innerHTML = "<label>Attachments</label>";
for (const f of msg.files) {
const fid = f.id || f.name;
const on = sel.files[fid] !== false;
const on = isOn && saved?.files?.[fid] !== false;
const url = fileUploadUrl(f);
const row = document.createElement("div");
row.className = "file-toggle-row";
@@ -876,10 +961,14 @@ function applyThreadImport(rootMsg, action) {
String(msg.ts) === rootTs ||
String(msg.thread_root_ts) === rootTs
) {
const sel = msgEntry(ch, msg.id);
sel.action = action;
sel.checked = action === "import";
if (action !== "merge") sel.merge_target = "";
if (action === "import" || action === "merge") {
const sel = msgEntry(ch, msg.id);
sel.action = action;
sel.checked = true;
if (action !== "merge") sel.merge_target = "";
} else {
clearMsgSelection(ch, msg.id);
}
}
}
if (channelEntry(ch).import_mode !== "pick") {
@@ -893,25 +982,28 @@ function applyThreadImport(rootMsg, action) {
function applyMessageDetailFromForm() {
if (!state.selectedChannel || !state.selectedMessageId) return;
const sel = msgEntry(state.selectedChannel, state.selectedMessageId);
sel.action = els.msgAction.value;
sel.merge_target =
els.msgAction.value === "merge" ? els.msgMergeTarget.value.trim() : "";
sel.notes = els.msgNotes.value;
sel.checked = sel.action === "import" || sel.action === "merge";
const action = els.msgAction.value;
const msg = state.allMessages.find((m) => m.id === state.selectedMessageId);
if (!msg) return;
els.fileToggles.querySelectorAll("[data-file-id]").forEach((inp) => {
sel.files[inp.dataset.fileId] = inp.checked;
});
const ch = channelEntry(state.selectedChannel);
if (sel.action !== "inherit") {
ch.import_mode = "pick";
ch.status = "partial";
ch.reviewed = true;
els.importModePick.checked = true;
if (action === "skip" || action === "inherit") {
clearMsgSelection(state.selectedChannel, state.selectedMessageId);
} else {
ensureCherryPick();
const sel = msgEntry(state.selectedChannel, state.selectedMessageId);
sel.action = action;
sel.checked = action === "import" || action === "merge";
sel.merge_target = action === "merge" ? els.msgMergeTarget.value.trim() : "";
sel.notes = els.msgNotes.value;
els.fileToggles.querySelectorAll("[data-file-id]").forEach((inp) => {
if (!sel.files) sel.files = {};
sel.files[inp.dataset.fileId] = inp.checked;
});
}
renderAllMessages();
renderSidebar();
updateTriageBar();
scheduleAutosave();
}
@@ -1038,60 +1130,166 @@ function skipChannel() {
persistNotes();
}
function syncGlobalSearchModeUi() {
const filesMode = els.globalSearchMode.value === "files";
els.globalFileType.classList.toggle("hidden", !filesMode);
els.globalSearch.placeholder = filesMode
? "Search attachments in all channels… (optional keywords)"
: "Search channels & messages… (press /)";
}
async function jumpToSearchHit(ch, msgId) {
els.globalSearchResults.classList.add("hidden");
els.globalSearch.value = "";
await selectChannel(ch);
if (msgId) {
const msg = state.allMessages.find((m) => m.id === msgId);
if (msg) {
requestAnimationFrame(() => {
focusMessage(msg.id);
openMessageDetail(msg);
document
.querySelector(
`[data-id="${msgId.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"]`
)
?.scrollIntoView({ block: "center" });
});
}
}
}
async function runGlobalSearch(query) {
if (!query.trim()) {
const filesMode = els.globalSearchMode.value === "files";
if (!filesMode && !query.trim()) {
els.globalSearchResults.classList.add("hidden");
return;
}
const data = await api(`/api/search?q=${encodeURIComponent(query)}&limit=60`);
const parts = [];
if (data.channels?.length) {
parts.push('<div class="search-group"><h4>Channels</h4>');
for (const ch of data.channels.slice(0, 12)) {
parts.push(
`<button type="button" class="search-hit" data-channel="${escapeHtml(ch.name)}">#${escapeHtml(ch.name)}</button>`
);
if (filesMode) {
const ft = els.globalFileType.value || "all";
const data = await api(
`/api/search-files?q=${encodeURIComponent(query)}&type=${encodeURIComponent(ft)}&limit=60`
);
if (data.hits?.length) {
const label =
ft === "all" ? "Attachments" : `${ft} files`;
parts.push(`<div class="search-group"><h4>${escapeHtml(label)}</h4>`);
for (const m of data.hits.slice(0, 25)) {
const names = (m.matching_files || m.files || [])
.map((f) => f.name)
.filter(Boolean)
.slice(0, 2)
.join(", ");
parts.push(
`<button type="button" class="search-hit" data-channel="${escapeHtml(m.channel)}" data-msg="${escapeHtml(m.id)}">
<strong>#${escapeHtml(m.channel)}</strong> · ${escapeHtml(m.user_name)}
<div class="sub">${escapeHtml(names || (m.text || "").slice(0, 80))}</div>
</button>`
);
}
parts.push("</div>");
}
parts.push("</div>");
}
if (data.messages?.length) {
parts.push('<div class="search-group"><h4>Messages</h4>');
for (const m of data.messages.slice(0, 20)) {
parts.push(
`<button type="button" class="search-hit" data-channel="${escapeHtml(m.channel)}" data-msg="${escapeHtml(m.id)}">
<strong>#${escapeHtml(m.channel)}</strong> · ${escapeHtml(m.user_name)}
<div class="sub">${escapeHtml((m.text || "").slice(0, 100))}</div>
</button>`
);
} else {
const data = await api(`/api/search?q=${encodeURIComponent(query)}&limit=60`);
if (data.channels?.length) {
parts.push('<div class="search-group"><h4>Channels</h4>');
for (const ch of data.channels.slice(0, 12)) {
parts.push(
`<button type="button" class="search-hit" data-channel="${escapeHtml(ch.name)}">#${escapeHtml(ch.name)}</button>`
);
}
parts.push("</div>");
}
if (data.messages?.length) {
parts.push('<div class="search-group"><h4>Messages</h4>');
for (const m of data.messages.slice(0, 20)) {
parts.push(
`<button type="button" class="search-hit" data-channel="${escapeHtml(m.channel)}" data-msg="${escapeHtml(m.id)}">
<strong>#${escapeHtml(m.channel)}</strong> · ${escapeHtml(m.user_name)}
<div class="sub">${escapeHtml((m.text || "").slice(0, 100))}</div>
</button>`
);
}
parts.push("</div>");
}
parts.push("</div>");
}
els.globalSearchResults.innerHTML =
parts.join("") || '<div class="empty" style="padding:1rem">No results</div>';
els.globalSearchResults.classList.remove("hidden");
els.globalSearchResults.querySelectorAll(".search-hit").forEach((btn) => {
btn.addEventListener("click", async () => {
const ch = btn.dataset.channel;
els.globalSearchResults.classList.add("hidden");
els.globalSearch.value = "";
await selectChannel(ch);
const msgId = btn.dataset.msg;
if (msgId) {
const msg = state.allMessages.find((m) => m.id === msgId);
if (msg) {
requestAnimationFrame(() => {
openMessageDetail(msg);
document.querySelector(`[data-id="${msgId}"]`)?.scrollIntoView({
block: "center",
});
});
}
}
});
btn.addEventListener("click", () => jumpToSearchHit(btn.dataset.channel, btn.dataset.msg));
});
}
function visibleFileMessages(imagesOnly) {
return visibleMessages().filter((msg) => {
const files = msg.files || [];
if (!files.length) return false;
if (imagesOnly && !files.some(isImageFile)) return false;
return true;
});
}
function selectVisibleFileMessages({ imagesOnly = false } = {}) {
if (!state.selectedChannel) return;
ensureCherryPick();
const targets = visibleFileMessages(imagesOnly);
const allSelected =
targets.length > 0 &&
targets.every((msg) => isMsgSelected(state.selectedChannel, msg));
let n = 0;
for (const msg of targets) {
const sel = msgEntry(state.selectedChannel, msg.id);
if (allSelected) {
clearMsgSelection(state.selectedChannel, msg.id);
} else {
setMsgImport(state.selectedChannel, msg, true);
const sel = msgEntry(state.selectedChannel, msg.id);
for (const f of msg.files || []) {
const fid = f.id || f.name;
if (!fid) continue;
if (imagesOnly && !isImageFile(f)) continue;
sel.files[fid] = true;
}
}
n += 1;
}
renderAllMessages();
renderSidebar();
updateTriageBar();
scheduleAutosave();
const label = imagesOnly ? "image" : "file";
const verb = allSelected ? "Cleared" : "Selected";
els.saveStatus.textContent = `${verb} ${n} ${label} message(s) in view`;
updateBulkSelectButtons();
setTimeout(() => {
els.saveStatus.textContent = "";
}, 2500);
}
function updateBulkSelectButtons() {
if (!els.btnSelectImages || !els.btnSelectFiles) return;
const imgTargets = visibleFileMessages(true);
const fileTargets = visibleFileMessages(false);
const imgAll =
imgTargets.length > 0 &&
imgTargets.every((m) => isMsgSelected(state.selectedChannel, m));
const fileAll =
fileTargets.length > 0 &&
fileTargets.every((m) => isMsgSelected(state.selectedChannel, m));
els.btnSelectImages.textContent = imgAll ? "Clear images in view" : "Select images in view";
els.btnSelectFiles.textContent = fileAll ? "Clear files in view" : "Select files in view";
els.btnSelectImages.classList.toggle("active", imgAll);
els.btnSelectFiles.classList.toggle("active", fileAll);
}
function selectVisibleImageMessages() {
selectVisibleFileMessages({ imagesOnly: true });
}
function scheduleAutosave() {
clearTimeout(state.saveTimer);
state.saveTimer = setTimeout(() => saveManifest(true), 600);
@@ -1131,11 +1329,15 @@ async function loadChannels() {
e.import = !!ch.import;
e.status = ch.status || (ch.import ? "import" : "undecided");
e.reviewed = !!ch.reviewed || ["import", "skip", "partial"].includes(e.status);
e.import_mode = ch.import_mode || "all";
e.import_mode =
ch.import_mode === "pick" || ch.status === "partial" ? "pick" : ch.import_mode || "all";
if (ch.status === "partial" && e.import_mode !== "pick") {
e.import_mode = "pick";
}
e.notes = ch.notes || "";
e.mattermost_channel = ch.mattermost_channel || "";
e.mattermost_team = ch.mattermost_team || "";
if (ch.message_selections) e.messages = ch.message_selections;
// Message picks come from loadManifest() only — avoid clobbering on parallel load.
}
}
@@ -1149,6 +1351,7 @@ async function loadManifest() {
function onMessageListClick(e) {
if (!state.selectedChannel) return;
if (e.target.closest(".msg-check")) return;
const previewBtn = e.target.closest(
".image-preview, .file-doc-preview, .file-open-link[data-preview-file-id], .file-open-link[data-preview-url]"
@@ -1165,27 +1368,6 @@ function onMessageListClick(e) {
return;
}
const check = e.target.closest("[data-msg-check]");
if (check) {
e.stopPropagation();
const row = check.closest(".message");
const msg = state.allMessages.find((m) => m.id === row?.dataset.id);
if (!msg) return;
const sel = msgEntry(state.selectedChannel, msg.id);
if (check.checked) {
sel.action = "import";
sel.checked = true;
} else {
sel.action = "skip";
sel.checked = false;
}
ensureCherryPick();
renderAllMessages();
scheduleAutosave();
focusMessage(msg.id, { scroll: false });
return;
}
const row = e.target.closest(".message");
if (!row) return;
const msg = state.allMessages.find((m) => m.id === row.dataset.id);
@@ -1212,6 +1394,7 @@ function bindEvents() {
[
els.messageSearch,
els.filterAttachmentType,
els.filterHideBots,
els.filterHideSystem,
els.filterOnlySelected,
@@ -1220,6 +1403,22 @@ function bindEvents() {
el.addEventListener("change", renderAllMessages);
});
els.btnSelectImages?.addEventListener("click", selectVisibleImageMessages);
els.btnSelectFiles?.addEventListener("click", () => selectVisibleFileMessages());
els.globalSearchMode.addEventListener("change", () => {
syncGlobalSearchModeUi();
const q = els.globalSearch.value;
if (els.globalSearchMode.value === "files" || q.trim()) {
runGlobalSearch(q);
} else {
els.globalSearchResults.classList.add("hidden");
}
});
els.globalFileType.addEventListener("change", () => {
if (els.globalSearchMode.value === "files") runGlobalSearch(els.globalSearch.value);
});
syncGlobalSearchModeUi();
els.importModePick.addEventListener("change", () => {
if (!state.selectedChannel) return;
const entry = channelEntry(state.selectedChannel);
@@ -1244,6 +1443,24 @@ function bindEvents() {
els.btnMsgPrev.addEventListener("click", goPrevMessage);
els.messages.addEventListener("click", onMessageListClick);
els.messages.addEventListener("change", (e) => {
const input = e.target.closest("input[data-msg-check]");
if (!input || !state.selectedChannel) return;
e.stopPropagation();
const row = input.closest(".message");
const msg = state.allMessages.find((m) => m.id === row?.dataset.id);
if (!msg) return;
setMsgImport(state.selectedChannel, msg, input.checked);
renderAllMessages();
renderSidebar();
updateTriageBar();
scheduleAutosave();
if (state.selectedMessageId === msg.id) {
openMessageDetail(msg);
} else {
focusMessage(msg.id, { scroll: false });
}
});
els.msgAction.addEventListener("change", () => {
els.msgMergeTarget.classList.toggle("hidden", els.msgAction.value !== "merge");
@@ -1266,7 +1483,7 @@ function bindEvents() {
els.globalSearch.addEventListener("input", () => {
clearTimeout(state.globalSearchTimer);
const q = els.globalSearch.value;
if (!q.trim()) {
if (!q.trim() && els.globalSearchMode.value !== "files") {
els.globalSearchResults.classList.add("hidden");
return;
}
@@ -1358,7 +1575,13 @@ async function init() {
bindEvents();
try {
const summary = await api("/api/summary");
await Promise.all([loadManifest(), loadChannels(), api("/api/users").then((d) => { state.users = d.users; })]);
await loadManifest();
await Promise.all([
loadChannels(),
api("/api/users").then((d) => {
state.users = d.users;
}),
]);
renderStats(summary);
renderSidebar();
renderSelectionSummary();