From 21780f91f3ccbe1a43e2f9eae24dee077daab5ec Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Tue, 19 May 2026 09:23:58 -0400
Subject: [PATCH 01/31] implemented state persistence

---
 .gitignore                           |   2 +-
 stable_pretraining/web/assets/app.js | 127 +++++++++++++++++++++++++++
 2 files changed, 128 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 8a56e6e09..fd2e692b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -161,7 +161,7 @@ multirun/
 ./data/
 examples/data/
 wandb/
-DISCOVERABILITY_PLAN.md
+SPT_WEB_PLAN.md
 
 *.ckpt
 *.pt
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 68b37e8fd..cf19aa829 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -79,6 +79,116 @@
     applyTheme(saved);
   }
 
+  // ---- state persistence -----------------------------------------------
+
+  function saveState() {
+    const snap = {
+      visible: [...state.visible],
+      filters: state.filters,
+      groupBy: state.groupBy,
+      sortBy: state.sortBy,
+      sortDesc: state.sortDesc,
+      xAxis: state.xAxis,
+      logY: state.logY,
+      smoothing: state.smoothing,
+      metricSearch: state.metricSearch,
+      activeTab: state.activeTab,
+      theme: state.theme,
+    };
+    try { localStorage.setItem('spt-web-state', JSON.stringify(snap)); } catch {}
+    const ids = [...state.visible].map(id => encodeURIComponent(id)).join(',');
+    try {
+      history.replaceState(null, '', '#runs=' + ids + '&tab=' + encodeURIComponent(state.activeTab));
+    } catch {}
+  }
+
+  function loadState() {
+    // Parse URL fragment first (wins over localStorage on conflict).
+    let fragVisible = null;
+    let fragTab = null;
+    if (location.hash.length > 1) {
+      const params = new URLSearchParams(location.hash.slice(1));
+      const runsParam = params.get('runs');
+      if (runsParam !== null) {
+        fragVisible = runsParam
+          ? runsParam.split(',').filter(Boolean).map(s => { try { return decodeURIComponent(s); } catch { return s; } })
+          : [];
+      }
+      const tabParam = params.get('tab');
+      if (tabParam && ['figures', 'out', 'err'].includes(tabParam)) fragTab = tabParam;
+    }
+
+    // Load localStorage snapshot.
+    let saved = null;
+    try {
+      const raw = localStorage.getItem('spt-web-state');
+      if (raw) saved = JSON.parse(raw);
+    } catch {}
+
+    if (saved && typeof saved === 'object') {
+      if (Array.isArray(saved.filters)) state.filters = saved.filters;
+      if (typeof saved.groupBy === 'string') state.groupBy = saved.groupBy;
+      if (typeof saved.sortBy === 'string') state.sortBy = saved.sortBy;
+      if (typeof saved.sortDesc === 'boolean') state.sortDesc = saved.sortDesc;
+      if (typeof saved.xAxis === 'string') state.xAxis = saved.xAxis;
+      if (typeof saved.logY === 'boolean') state.logY = saved.logY;
+      if (typeof saved.smoothing === 'number') state.smoothing = saved.smoothing;
+      if (typeof saved.metricSearch === 'string') state.metricSearch = saved.metricSearch;
+      if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
+      if (typeof saved.theme === 'string') state.theme = saved.theme;
+      if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
+    }
+
+    // Fragment overrides localStorage for visible runs and active tab.
+    if (fragVisible !== null) state._pendingVisible = new Set(fragVisible);
+    if (fragTab !== null) state.activeTab = fragTab;
+
+    // Keep the legacy spt-web-theme key in sync so initTheme() still works.
+    try { localStorage.setItem('spt-web-theme', state.theme); } catch {}
+  }
+
+  function syncControlsToState() {
+    const smEl = document.getElementById('smoothing');
+    const smvEl = document.getElementById('smoothing-val');
+    if (smEl) smEl.value = String(state.smoothing);
+    if (smvEl) smvEl.textContent = state.smoothing.toFixed(2);
+    const xEl = document.getElementById('x-axis');
+    if (xEl) xEl.value = state.xAxis;
+    const lyEl = document.getElementById('log-y');
+    if (lyEl) lyEl.checked = state.logY;
+    const msEl = document.getElementById('metric-search');
+    if (msEl) msEl.value = state.metricSearch;
+    // Apply active tab to DOM directly — calling setActiveTab() would fire
+    // saveState() while state.visible is still empty (runs not yet loaded).
+    for (const btn of document.querySelectorAll('#tabs .tab')) {
+      btn.classList.toggle('active', btn.dataset.tab === state.activeTab);
+    }
+    for (const pane of document.querySelectorAll('.tab-pane')) {
+      pane.classList.toggle('active', pane.id === `tab-${state.activeTab}`);
+    }
+  }
+
+  async function applyPendingVisible() {
+    if (!state._pendingVisible) return;
+    const ids = [...state._pendingVisible].filter(id => state.runs.has(id));
+    state._pendingVisible = null;
+    if (!ids.length) return;
+    for (const id of ids) state.visible.add(id);
+    const needMetrics = ids.filter(id => !state.metrics.has(id));
+    const needMedia = ids.filter(id => {
+      if (state.media.has(id)) return false;
+      const r = state.runs.get(id);
+      return r && r.has_media;
+    });
+    await Promise.all([...needMetrics.map(fetchMetrics), ...needMedia.map(fetchMedia)]);
+    renderRunList();
+    scheduleRerender();
+    if (state.activeTab === 'out' || state.activeTab === 'err') {
+      await refreshLogStreamsForVisibleRuns();
+      renderLogTab(state.activeTab);
+    }
+  }
+
   // ---- color: stable hash → HSL ----------------------------------------
 
   const colorCache = new Map();
@@ -125,6 +235,7 @@
     }
     renderRunList();
     updateHeaderStats();
+    await applyPendingVisible();
   }
 
   // Stream-fetch metrics via NDJSON. The server emits one JSON object per
@@ -438,6 +549,7 @@
         renderFilters();
         renderRunList();
         scheduleRerender();
+        saveState();
       });
       chip.appendChild(rm);
 
@@ -524,6 +636,7 @@
       renderFilters();
       renderRunList();
       scheduleRerender();
+      saveState();
     });
   }
 
@@ -641,6 +754,7 @@
       state.visible.delete(id);
       renderRunList();
       scheduleRerender();
+      saveState();
       return;
     }
     state.visible.add(id);
@@ -664,6 +778,7 @@
     }
     renderRunList();
     scheduleRerender();
+    saveState();
     if (state.activeTab === 'out' || state.activeTab === 'err') {
       // Selection changed → log-stream options change.
       fetchLogStreams(id).then(() => renderLogTab(state.activeTab));
@@ -692,6 +807,7 @@
     }
     renderRunList();
     scheduleRerender();
+    saveState();
     if (state.activeTab === 'out' || state.activeTab === 'err') {
       refreshLogStreamsForVisibleRuns().then(() => renderLogTab(state.activeTab));
     }
@@ -1929,6 +2045,7 @@
     for (const pane of document.querySelectorAll('.tab-pane')) {
       pane.classList.toggle('active', pane.id === `tab-${name}`);
     }
+    saveState();
     if (name === 'out' || name === 'err') {
       // Lazily fetch logs the first time the user opens a log tab.
       refreshLogStreamsForVisibleRuns().then(() => renderLogTab(name));
@@ -2050,16 +2167,19 @@
       state.smoothing = parseFloat(e.target.value);
       smv.textContent = state.smoothing.toFixed(2);
       scheduleRerender();
+      saveState();
     });
 
     document.getElementById('x-axis').addEventListener('change', e => {
       state.xAxis = e.target.value;
       scheduleRerender();
+      saveState();
     });
 
     document.getElementById('log-y').addEventListener('change', e => {
       state.logY = e.target.checked;
       scheduleRerender();
+      saveState();
     });
 
     document.getElementById('select-all').addEventListener('click', () => setAllVisible(true));
@@ -2072,6 +2192,7 @@
       renderRunList();
       // Group key shows up in chart legends → rebuild charts.
       scheduleRerender();
+      saveState();
     });
 
     // Tabs
@@ -2104,6 +2225,7 @@
         state.metricSearch = e.target.value;
         state._lastTreeKey = null; // force rebuild
         scheduleRerender();
+        saveState();
       }, 80));
     }
 
@@ -2127,11 +2249,13 @@
 
     document.getElementById('theme-toggle').addEventListener('click', () => {
       applyTheme(state.theme === 'dark' ? 'light' : 'dark');
+      saveState();
     });
 
     document.getElementById('sort-by').addEventListener('change', e => {
       state.sortBy = e.target.value;
       renderRunList();
+      saveState();
     });
 
     const sortDirBtn = document.getElementById('sort-dir');
@@ -2140,6 +2264,7 @@
       state.sortDesc = !state.sortDesc;
       sortDirBtn.textContent = state.sortDesc ? '↓' : '↑';
       renderRunList();
+      saveState();
     });
 
     // Detail modal wiring.
@@ -2203,7 +2328,9 @@
   // ---- init -------------------------------------------------------------
 
   async function main() {
+    loadState();
     wireControls();
+    syncControlsToState();
     initTheme();
     // Start SSE first so we don't miss progress events emitted between the
     // status fetch and the first scan-tick.

From 7df08bd27de6c7de2664f4969dfc31876d8e29bb Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 17:26:08 -0400
Subject: [PATCH 02/31] many new features for spt web, more coming soon

---
 .gitignore                               |   3 +
 stable_pretraining/web/assets/app.css    | 157 +++++++++++++
 stable_pretraining/web/assets/app.js     | 275 ++++++++++++++++++++++-
 stable_pretraining/web/assets/index.html |   8 +
 4 files changed, 438 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index fd2e692b7..52395a462 100644
--- a/.gitignore
+++ b/.gitignore
@@ -159,9 +159,12 @@ docs/source/modules.rst
 outputs/
 multirun/
 ./data/
+mock_runs/
 examples/data/
 wandb/
 SPT_WEB_PLAN.md
+scripts/
+.DS_Store
 
 *.ckpt
 *.pt
diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 973774ec2..8f35f087b 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -579,6 +579,163 @@ body {
 }
 .chart-body { width: 100%; }
 
+/* ---- chart annotations ------------------------------------------------ */
+.chart-annot {
+  margin-top: 6px;
+  border-top: 1px solid var(--border-2);
+  padding-top: 4px;
+}
+.chart-annot:empty { display: none; }
+.chart-annot-table { display: contents; }
+.chart-annot-row {
+  display: grid;
+  grid-template-columns: 10px 1fr auto auto;
+  gap: 6px;
+  align-items: center;
+  padding: 2px 4px;
+  border-radius: 3px;
+  font-size: 11px;
+  font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+  color: var(--muted-2);
+}
+.chart-annot-header {
+  color: var(--muted-3);
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.4px;
+  cursor: default;
+}
+.chart-annot-row:not(.chart-annot-header) { cursor: pointer; }
+.chart-annot-row:not(.chart-annot-header):hover { background: var(--hover); }
+.chart-annot-row.annot-best { color: var(--text); }
+.annot-dot {
+  width: 8px; height: 8px;
+  border-radius: 2px;
+  display: inline-block;
+  flex-shrink: 0;
+}
+.annot-label {
+  overflow: hidden; white-space: nowrap; text-overflow: ellipsis;
+  min-width: 0;
+}
+.annot-val { text-align: right; white-space: nowrap; }
+.annot-best-bold { font-weight: 700; color: var(--text-strong); }
+
+/* ---- runs table ------------------------------------------------------- */
+
+#table-controls {
+  display: flex; align-items: center; gap: 10px;
+  padding: 8px 12px;
+  border-bottom: 1px solid var(--border);
+  flex: 0 0 auto;
+}
+#table-col-search {
+  flex: 1 1 220px;
+  max-width: 320px;
+  padding: 4px 8px;
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  font-size: 12px;
+  color: var(--text);
+}
+#table-col-search:focus { outline: 1px solid var(--accent); }
+.table-col-count {
+  font-size: 11px;
+  color: var(--muted-2);
+  white-space: nowrap;
+}
+.runs-table-wrap {
+  flex: 1 1 auto;
+  overflow: auto;
+  padding: 0 12px 12px;
+}
+.runs-table {
+  border-collapse: separate;
+  border-spacing: 0;
+  font-size: 12px;
+  font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+  white-space: nowrap;
+  width: max-content;
+  min-width: 100%;
+}
+.runs-table thead th {
+  position: sticky;
+  top: 0;
+  background: var(--surface-2);
+  border-bottom: 1px solid var(--border);
+  padding: 6px 10px;
+  text-align: left;
+  font-weight: 600;
+  color: var(--muted);
+  font-size: 11px;
+  z-index: 2;
+  cursor: pointer;
+  user-select: none;
+}
+.runs-table thead th:hover { color: var(--text); }
+.runs-table thead th.rt-sorted { color: var(--accent); }
+.runs-table thead th.col-diff { color: var(--text-strong); }
+.rt-corner {
+  position: sticky !important;
+  left: 0;
+  z-index: 4 !important;
+  cursor: default !important;
+  min-width: 140px;
+}
+.rt-col-hdr { min-width: 80px; max-width: 200px; }
+.rt-col-label {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  max-width: 160px;
+  display: inline-block;
+  vertical-align: middle;
+}
+.rt-sort-arrow { margin-left: 3px; color: var(--accent); }
+.rt-section-start { border-left: 2px solid var(--border-strong); }
+.runs-table tbody tr {
+  cursor: pointer;
+}
+.runs-table tbody tr:hover td { background: var(--hover); }
+.runs-table tbody tr:hover td.rt-run-id { background: var(--hover-2); }
+.runs-table tbody td {
+  padding: 5px 10px;
+  border-bottom: 1px solid var(--border-2);
+  color: var(--muted-2);
+}
+.runs-table tbody td.cell-diff {
+  color: var(--text-strong);
+  background: rgba(251, 191, 36, 0.07);
+}
+[data-theme="light"] .runs-table tbody td.cell-diff {
+  background: rgba(180, 83, 9, 0.06);
+}
+.runs-table tbody td.cell-same { color: var(--muted-3); }
+.rt-run-id {
+  position: sticky;
+  left: 0;
+  background: var(--surface);
+  z-index: 1;
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  min-width: 140px;
+  font-weight: 500;
+  color: var(--text);
+  border-right: 1px solid var(--border);
+}
+.rt-dot {
+  width: 9px; height: 9px;
+  border-radius: 2px;
+  flex-shrink: 0;
+  border: 1px solid var(--dot-border);
+}
+.rt-run-label {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  max-width: 120px;
+}
+
 /* ---- media panels --------------------------------------------------- */
 
 .media-panel .chart-title { gap: 10px; }
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index cf19aa829..e2ebc3b1e 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -26,8 +26,10 @@
     expandedMetrics: new Set(),  // metric tree paths the user has expanded (closed by default)
     openSidebarGroups: new Set(), // sidebar group keys the user has expanded
     theme: 'dark',
-    activeTab: 'figures',   // 'figures' | 'out' | 'err'
+    activeTab: 'figures',   // 'figures' | 'out' | 'err' | 'table'
     metricSearch: '',
+    tableColSearch: '',
+    tableSort: null,         // null | { col: string, dir: 'asc' | 'desc' }
     // Cached per-run log discoveries: runId -> {streams, fetchedAt}
     logsIndex: new Map(),
     // Currently selected (runId, stream_id) for each kind, persisted across
@@ -92,6 +94,7 @@
       logY: state.logY,
       smoothing: state.smoothing,
       metricSearch: state.metricSearch,
+      tableColSearch: state.tableColSearch,
       activeTab: state.activeTab,
       theme: state.theme,
     };
@@ -115,7 +118,7 @@
           : [];
       }
       const tabParam = params.get('tab');
-      if (tabParam && ['figures', 'out', 'err'].includes(tabParam)) fragTab = tabParam;
+      if (tabParam && ['figures', 'out', 'err', 'table'].includes(tabParam)) fragTab = tabParam;
     }
 
     // Load localStorage snapshot.
@@ -134,6 +137,7 @@
       if (typeof saved.logY === 'boolean') state.logY = saved.logY;
       if (typeof saved.smoothing === 'number') state.smoothing = saved.smoothing;
       if (typeof saved.metricSearch === 'string') state.metricSearch = saved.metricSearch;
+      if (typeof saved.tableColSearch === 'string') state.tableColSearch = saved.tableColSearch;
       if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
       if (typeof saved.theme === 'string') state.theme = saved.theme;
       if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
@@ -158,6 +162,8 @@
     if (lyEl) lyEl.checked = state.logY;
     const msEl = document.getElementById('metric-search');
     if (msEl) msEl.value = state.metricSearch;
+    const tcsEl = document.getElementById('table-col-search');
+    if (tcsEl) tcsEl.value = state.tableColSearch;
     // Apply active tab to DOM directly — calling setActiveTab() would fire
     // saveState() while state.visible is still empty (runs not yet loaded).
     for (const btn of document.querySelectorAll('#tabs .tab')) {
@@ -822,6 +828,7 @@
     requestAnimationFrame(() => {
       renderPending = false;
       renderCharts();
+      if (state.activeTab === 'table') renderRunsTable();
     });
   }
 
@@ -1017,6 +1024,176 @@
     for (const tag of mediaTags.keys()) updateMediaPanel(tag);
   }
 
+  // ---- runs table ---------------------------------------------------------
+
+  function formatCellValue(v) {
+    if (v == null) return '';
+    if (typeof v === 'boolean') return String(v);
+    if (typeof v === 'number') {
+      if (!isFinite(v)) return String(v);
+      if (Number.isInteger(v)) return String(v);
+      const abs = Math.abs(v);
+      if (abs !== 0 && (abs < 1e-3 || abs >= 1e6)) return v.toExponential(2);
+      return String(+v.toPrecision(4));
+    }
+    const s = String(v);
+    return s.length > 30 ? s.slice(0, 28) + '…' : s;
+  }
+
+  function renderRunsTable() {
+    const wrap = document.querySelector('#tab-table .runs-table-wrap');
+    if (!wrap) return;
+
+    const visIds = effectivelyVisible();
+    const runs = visIds.map(id => state.runs.get(id)).filter(Boolean);
+
+    const cnt = document.getElementById('table-col-count');
+
+    if (runs.length === 0) {
+      wrap.replaceChildren();
+      const empty = document.createElement('div');
+      empty.className = 'empty-state';
+      empty.textContent = 'select runs on the left to compare them in the table';
+      wrap.appendChild(empty);
+      if (cnt) cnt.textContent = '';
+      return;
+    }
+
+    // Build union of hparam + summary column keys across all visible runs.
+    const hparamKeys = new Set();
+    const summaryKeys = new Set();
+    for (const r of runs) {
+      for (const k of Object.keys(r.hparams || {})) hparamKeys.add(k);
+      for (const k of Object.keys(r.summary || {})) summaryKeys.add(k);
+    }
+    const allCols = [
+      ...[...hparamKeys].sort().map(k => ({ id: `hparams.${k}`, label: k, section: 'hparams' })),
+      ...[...summaryKeys].sort().map(k => ({ id: `summary.${k}`, label: k, section: 'summary' })),
+    ];
+
+    // Filter columns by search.
+    const q = (state.tableColSearch || '').trim().toLowerCase();
+    const cols = q ? allCols.filter(c => c.label.toLowerCase().includes(q)) : allCols;
+
+    if (cnt) {
+      cnt.textContent = cols.length === allCols.length
+        ? `${allCols.length} cols`
+        : `${cols.length} / ${allCols.length} cols`;
+    }
+
+    // Determine which columns have any differing values.
+    const diffSet = new Set();
+    for (const col of cols) {
+      const vals = runs.map(r => {
+        const v = valueAt(r, col.id);
+        return v == null ? '\x00null' : String(v);
+      });
+      if (new Set(vals).size > 1) diffSet.add(col.id);
+    }
+
+    // Apply table sort.
+    let sortedRuns = [...runs];
+    const ts = state.tableSort;
+    if (ts) {
+      sortedRuns.sort((a, b) => {
+        const c = compareValues(valueAt(a, ts.col), valueAt(b, ts.col));
+        return ts.dir === 'asc' ? c : -c;
+      });
+    }
+
+    // Track the first column of each section for a visual separator.
+    const sectionStarts = new Set();
+    let lastSection = null;
+    for (const col of cols) {
+      if (col.section !== lastSection) { sectionStarts.add(col.id); lastSection = col.section; }
+    }
+
+    // Build table.
+    const table = document.createElement('table');
+    table.className = 'runs-table';
+
+    // Header row.
+    const thead = document.createElement('thead');
+    const hdrRow = document.createElement('tr');
+
+    const cornerTh = document.createElement('th');
+    cornerTh.className = 'rt-corner';
+    cornerTh.textContent = 'run';
+    hdrRow.appendChild(cornerTh);
+
+    for (const col of cols) {
+      const th = document.createElement('th');
+      const cls = ['rt-col-hdr', diffSet.has(col.id) ? 'col-diff' : 'col-same'];
+      if (sectionStarts.has(col.id)) cls.push('rt-section-start');
+      if (ts && ts.col === col.id) cls.push('rt-sorted');
+      th.className = cls.join(' ');
+      th.title = col.id;
+
+      const inner = document.createElement('span');
+      inner.className = 'rt-col-label';
+      inner.textContent = col.label;
+      th.appendChild(inner);
+
+      if (ts && ts.col === col.id) {
+        const arrow = document.createElement('span');
+        arrow.className = 'rt-sort-arrow';
+        arrow.textContent = ts.dir === 'asc' ? '↑' : '↓';
+        th.appendChild(arrow);
+      }
+
+      th.addEventListener('click', () => {
+        if (ts && ts.col === col.id) {
+          state.tableSort = ts.dir === 'asc' ? { col: col.id, dir: 'desc' } : null;
+        } else {
+          state.tableSort = { col: col.id, dir: 'asc' };
+        }
+        renderRunsTable();
+      });
+
+      hdrRow.appendChild(th);
+    }
+
+    thead.appendChild(hdrRow);
+    table.appendChild(thead);
+
+    // Body rows.
+    const tbody = document.createElement('tbody');
+    for (const run of sortedRuns) {
+      const tr = document.createElement('tr');
+      tr.addEventListener('click', () => openDetail(run.run_id));
+
+      // Sticky run-ID cell.
+      const idTd = document.createElement('td');
+      idTd.className = 'rt-run-id';
+      const dot = document.createElement('span');
+      dot.className = 'rt-dot';
+      dot.style.background = runColor(run.run_id);
+      const lbl = document.createElement('span');
+      lbl.className = 'rt-run-label';
+      lbl.textContent = run.run_id.split('/').pop() || run.run_id;
+      lbl.title = run.run_id;
+      idTd.append(dot, lbl);
+      tr.appendChild(idTd);
+
+      // Data cells.
+      for (const col of cols) {
+        const td = document.createElement('td');
+        const cls = [diffSet.has(col.id) ? 'cell-diff' : 'cell-same'];
+        if (sectionStarts.has(col.id)) cls.push('rt-section-start');
+        td.className = cls.join(' ');
+        const val = valueAt(run, col.id);
+        td.textContent = formatCellValue(val);
+        td.title = val != null ? String(val) : '';
+        tr.appendChild(td);
+      }
+
+      tbody.appendChild(tr);
+    }
+    table.appendChild(tbody);
+
+    wrap.replaceChildren(table);
+  }
+
   function makePanel(fullName, displayName) {
     const panel = document.createElement('div');
     panel.className = 'chart-panel';
@@ -1031,6 +1208,9 @@
     const body = document.createElement('div');
     body.className = 'chart-body';
     panel.appendChild(body);
+    const annot = document.createElement('div');
+    annot.className = 'chart-annot';
+    panel.appendChild(annot);
     return panel;
   }
 
@@ -1325,7 +1505,19 @@
         spanGaps: true,
       });
     }
-    return { data, seriesCfg, runIds: series.map(s => s.id) };
+    const lower = lowerIsBetter(name);
+    const seriesStats = series.map(s => {
+      const raw = s.ys.filter(v => v != null && isFinite(v));
+      const lastVal = raw.length ? raw[raw.length - 1] : null;
+      const bestVal = raw.length ? (lower ? Math.min(...raw) : Math.max(...raw)) : null;
+      return { id: s.id, lastVal, bestVal };
+    });
+    return { data, seriesCfg, runIds: series.map(s => s.id), seriesStats };
+  }
+
+  function lowerIsBetter(name) {
+    const n = name.toLowerCase();
+    return n.includes('loss') || n.includes('err') || n.includes('perplexity') || n.includes('ppl');
   }
 
   function fmtTooltipNum(v) {
@@ -1477,12 +1669,70 @@
     };
   }
 
+  function updateAnnotTable(name, entry, seriesCfg, seriesStats) {
+    const annot = entry.panel.querySelector('.chart-annot');
+    if (!annot) return;
+    if (!seriesStats || seriesStats.length === 0) { annot.replaceChildren(); return; }
+
+    const lower = lowerIsBetter(name);
+    const validBests = seriesStats.map(s => s.bestVal).filter(v => v != null);
+    const overallBest = validBests.length
+      ? (lower ? Math.min(...validBests) : Math.max(...validBests))
+      : null;
+
+    annot.replaceChildren();
+
+    // Header row
+    const hdr = document.createElement('div');
+    hdr.className = 'chart-annot-row chart-annot-header';
+    hdr.innerHTML = '<span></span><span></span><span>last</span><span>' + (lower ? 'min' : 'max') + '</span>';
+    annot.appendChild(hdr);
+
+    for (let i = 0; i < seriesStats.length; i++) {
+      const { id, lastVal, bestVal } = seriesStats[i];
+      const seriesIdx = i + 1;
+      const s = seriesCfg[seriesIdx];
+      if (!s) continue;
+      const color = typeof s.stroke === 'function' ? s.stroke() : s.stroke;
+      const isBest = bestVal != null && bestVal === overallBest;
+
+      const row = document.createElement('div');
+      row.className = 'chart-annot-row' + (isBest ? ' annot-best' : '');
+      row.title = id;
+
+      const dot = document.createElement('span');
+      dot.className = 'annot-dot';
+      dot.style.background = color;
+
+      const label = document.createElement('span');
+      label.className = 'annot-label';
+      label.textContent = id.split('/').pop() || id;
+
+      const lastEl = document.createElement('span');
+      lastEl.className = 'annot-val';
+      lastEl.textContent = fmtTooltipNum(lastVal);
+
+      const bestEl = document.createElement('span');
+      bestEl.className = isBest ? 'annot-val annot-best-bold' : 'annot-val';
+      bestEl.textContent = fmtTooltipNum(bestVal);
+
+      row.append(dot, label, lastEl, bestEl);
+      row.addEventListener('click', () => {
+        if (!entry.plot) return;
+        const cur = entry.plot.series[seriesIdx];
+        if (!cur) return;
+        entry.plot.setSeries(seriesIdx, { show: cur.show === false });
+      });
+      annot.appendChild(row);
+    }
+  }
+
   function updateChart(name) {
     const entry = state.charts.get(name);
     if (!entry) return;
 
     const visibleIds = effectivelyVisible().sort();
-    const { data, seriesCfg, runIds } = buildChartData(name, visibleIds);
+    const { data, seriesCfg, runIds, seriesStats } = buildChartData(name, visibleIds);
 
     if (runIds.length === 0) {
       // No data yet for this chart with current selection; clear.
@@ -1491,6 +1741,7 @@
         entry.plot = null;
         entry.configKey = '';
       }
+      updateAnnotTable(name, entry, [], []);
       return;
     }
 
@@ -1498,6 +1749,7 @@
 
     if (entry.plot && entry.configKey === configKey) {
       entry.plot.setData(data);
+      updateAnnotTable(name, entry, seriesCfg, seriesStats);
       return;
     }
 
@@ -1506,6 +1758,7 @@
     const width = body.clientWidth || 400;
     entry.plot = new uPlot(makeUplotOpts(name, width, seriesCfg), data, body);
     entry.configKey = configKey;
+    updateAnnotTable(name, entry, seriesCfg, seriesStats);
   }
 
   // ---- landing / overview ---------------------------------------------
@@ -2037,7 +2290,7 @@
   // ---- tabs -------------------------------------------------------------
 
   function setActiveTab(name) {
-    if (!['figures', 'out', 'err'].includes(name)) return;
+    if (!['figures', 'out', 'err', 'table'].includes(name)) return;
     state.activeTab = name;
     for (const btn of document.querySelectorAll('#tabs .tab')) {
       btn.classList.toggle('active', btn.dataset.tab === name);
@@ -2049,6 +2302,8 @@
     if (name === 'out' || name === 'err') {
       // Lazily fetch logs the first time the user opens a log tab.
       refreshLogStreamsForVisibleRuns().then(() => renderLogTab(name));
+    } else if (name === 'table') {
+      renderRunsTable();
     }
   }
 
@@ -2229,6 +2484,16 @@
       }, 80));
     }
 
+    // Table column search
+    const tcs = document.getElementById('table-col-search');
+    if (tcs) {
+      tcs.addEventListener('input', debounce(e => {
+        state.tableColSearch = e.target.value;
+        saveState();
+        renderRunsTable();
+      }, 80));
+    }
+
     // Log tab selectors / refresh buttons
     const outSel = document.getElementById('logs-stream-selector');
     if (outSel) outSel.addEventListener('change', () => {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index e8a0149c4..86e96f5fe 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -52,6 +52,7 @@
       <button class="tab active" data-tab="figures" type="button">figures</button>
       <button class="tab" data-tab="out" type="button">.out</button>
       <button class="tab" data-tab="err" type="button">.err</button>
+      <button class="tab" data-tab="table" type="button">table</button>
     </nav>
     <div id="tab-figures" class="tab-pane active">
       <div id="figures-controls">
@@ -90,6 +91,13 @@
       </div>
       <pre id="logs-err" class="log-view"></pre>
     </div>
+    <div id="tab-table" class="tab-pane">
+      <div id="table-controls">
+        <input id="table-col-search" type="search" placeholder="filter columns..." autocomplete="off" spellcheck="false">
+        <span id="table-col-count" class="table-col-count"></span>
+      </div>
+      <div class="runs-table-wrap"></div>
+    </div>
   </main>
 </div>
 <div id="image-lightbox" class="lightbox" hidden role="dialog" aria-modal="true" aria-label="image preview">

From 985605af36ee4d0e8be60bc229a05c231dd4721d Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 19:17:44 -0400
Subject: [PATCH 03/31] making it possible to update the names internally and
 have persistence

---
 stable_pretraining/web/assets/app.css | 23 +++++++-
 stable_pretraining/web/assets/app.js  | 79 +++++++++++++++++++++++----
 stable_pretraining/web/scan.py        | 55 +++++++++++++++++++
 stable_pretraining/web/server.py      | 42 ++++++++++++++
 4 files changed, 187 insertions(+), 12 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 8f35f087b..871329caa 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -253,10 +253,29 @@ body {
   50%      { opacity: 1.0; }
 }
 .run-name {
-  flex: 1; overflow: hidden;
-  white-space: nowrap; text-overflow: ellipsis;
+  flex: 1; overflow: hidden; min-width: 0;
+  display: flex; flex-direction: column; gap: 1px;
+}
+.run-display-name {
+  overflow: hidden; white-space: nowrap; text-overflow: ellipsis;
+  font-variant-numeric: tabular-nums;
+  cursor: text;
+}
+.run-id-hint {
+  font-size: 10px; color: var(--muted-3);
+  overflow: hidden; white-space: nowrap; text-overflow: ellipsis;
   font-variant-numeric: tabular-nums;
 }
+.run-name-edit {
+  width: 100%; min-width: 0; box-sizing: border-box;
+  background: var(--surface-2);
+  border: 1px solid var(--accent);
+  border-radius: 3px;
+  color: var(--text);
+  font-size: 12px; font-family: inherit;
+  padding: 1px 4px;
+  outline: none;
+}
 .run-status {
   font-size: 10px; color: var(--muted);
   text-transform: uppercase; letter-spacing: 0.5px;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index e2ebc3b1e..2b3ee7e42 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -224,6 +224,16 @@
     return r.json();
   }
 
+  async function patchRunMeta(runId, fields) {
+    const res = await fetch('/api/run-meta', {
+      method: 'PATCH',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ run_id: runId, ...fields }),
+    });
+    if (!res.ok) throw new Error(await res.text().catch(() => String(res.status)));
+    return res.json();
+  }
+
   async function refreshRuns() {
     const runs = await fetchJSON('/api/runs');
     const seen = new Set();
@@ -661,11 +671,60 @@
     dot.style.background = runColor(r.run_id);
     row.appendChild(dot);
 
-    const name = document.createElement('div');
-    name.className = 'run-name';
-    name.title = `${r.run_id}\n${r.run_dir || ''}`;
-    name.textContent = r.run_id;
-    row.appendChild(name);
+    // Name column: primary display_name + optional dimmed run_id hint.
+    const nameWrap = document.createElement('div');
+    nameWrap.className = 'run-name';
+
+    const displayEl = document.createElement('div');
+    displayEl.className = 'run-display-name';
+    displayEl.textContent = r.display_name || r.run_id;
+    displayEl.title = `${r.run_id}\n${r.run_dir || ''}`;
+    nameWrap.appendChild(displayEl);
+
+    // Show run_id as a hint only when it differs from the display name
+    // (i.e. the path has been shortened, or a custom name was set).
+    if (r.display_name && r.display_name !== r.run_id) {
+      const hint = document.createElement('div');
+      hint.className = 'run-id-hint';
+      hint.textContent = r.run_id;
+      nameWrap.appendChild(hint);
+    }
+
+    // Double-click on the display name starts inline editing.
+    displayEl.addEventListener('dblclick', e => {
+      e.stopPropagation();
+      const prev = r.display_name || r.run_id;
+      const input = document.createElement('input');
+      input.className = 'run-name-edit';
+      input.value = prev;
+      displayEl.replaceWith(input);
+      input.focus();
+      input.select();
+
+      let done = false;
+      function commit() {
+        if (done) return;
+        done = true;
+        const val = input.value.trim();
+        input.replaceWith(displayEl);
+        if (val && val !== prev) {
+          displayEl.textContent = val;
+          const run = state.runs.get(r.run_id);
+          if (run) run.display_name = val;
+          patchRunMeta(r.run_id, { display_name: val }).catch(() => {
+            displayEl.textContent = prev;
+            if (run) run.display_name = prev;
+          });
+        }
+      }
+      input.addEventListener('blur', commit);
+      input.addEventListener('keydown', ke => {
+        if (ke.key === 'Enter') { ke.preventDefault(); input.blur(); }
+        if (ke.key === 'Escape') { done = true; input.replaceWith(displayEl); }
+      });
+    });
+
+    row.appendChild(nameWrap);
 
     if (r.status) {
       const st = document.createElement('div');
@@ -709,8 +768,8 @@
     // that affect appearance (visibility, loading-pulse) are repainted by
     // touching only the affected row's classList below.
     const layoutKey = JSON.stringify({
-      groupKeys: groups.map(g => [g.key, g.runs.map(r => r.run_id)]),
-      ungrouped: ungrouped.map(r => r.run_id),
+      groupKeys: groups.map(g => [g.key, g.runs.map(r => [r.run_id, r.display_name])]),
+      ungrouped: ungrouped.map(r => [r.run_id, r.display_name]),
       open: [...state.openSidebarGroups].sort(),
     });
     if (state._lastListKey === layoutKey) {
@@ -1170,7 +1229,7 @@
       dot.style.background = runColor(run.run_id);
       const lbl = document.createElement('span');
       lbl.className = 'rt-run-label';
-      lbl.textContent = run.run_id.split('/').pop() || run.run_id;
+      lbl.textContent = run.display_name || run.run_id;
       lbl.title = run.run_id;
       idTd.append(dot, lbl);
       tr.appendChild(idTd);
@@ -1880,7 +1939,7 @@
     row.appendChild(dot);
     const name = document.createElement('div');
     name.className = 'recent-run-name';
-    name.textContent = r.run_id;
+    name.textContent = r.display_name || r.run_id;
     name.title = `${r.run_id}\n${r.run_dir || ''}`;
     row.appendChild(name);
     if (r.status) {
@@ -2153,7 +2212,7 @@
     if (!r) return closeDetail();
 
     document.getElementById('detail-dot').style.background = runColor(r.run_id);
-    document.getElementById('detail-title').textContent = r.run_id;
+    document.getElementById('detail-title').textContent = r.display_name || r.run_id;
 
     const meta = {
       run_dir: r.run_dir,
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index d1720b479..8b1bd8c98 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -21,6 +21,8 @@
 from pathlib import Path
 from typing import Any, Optional
 
+from ..registry._sidecar import SIDECAR_NAME, write_sidecar
+
 
 @dataclass
 class _Run:
@@ -388,9 +390,17 @@ def runs_json(self) -> list[dict]:
     @staticmethod
     def _serialize(run: _Run) -> dict:
         s = run.sidecar
+        # display_name: explicit sidecar field wins; fall back to the last
+        # path component of run_id so short IDs stay readable by default.
+        display_name = (
+            s.get("display_name")
+            or run.run_id.rsplit("/", 1)[-1]
+            or run.run_id
+        )
         return {
             "run_id": run.run_id,
             "run_dir": str(run.run_dir),
+            "display_name": display_name,
             "status": s.get("status"),
             "created_at": s.get("created_at"),
             "tags": s.get("tags") or [],
@@ -717,6 +727,51 @@ def media_json(self, run_id: str) -> Optional[dict]:
             pass
         return {"events": events}
 
+    _MUTABLE_FIELDS = frozenset({"display_name", "notes", "tags", "archived"})
+
+    def patch_run_meta(self, run_id: str, patch: dict) -> bool:
+        """Atomically apply a metadata patch to a run's sidecar.
+
+        Args:
+            run_id: The run to update (must be known to the scanner).
+            patch: Dict containing any subset of ``display_name``, ``notes``,
+                ``tags``, and ``archived``. Other keys raise ``ValueError``.
+
+        Returns:
+            ``True`` on success, ``False`` if ``run_id`` is not known.
+
+        Raises:
+            ValueError: If ``patch`` contains fields outside the mutable set.
+        """
+        bad = set(patch) - self._MUTABLE_FIELDS
+        if bad:
+            raise ValueError(f"unknown patch fields: {sorted(bad)}")
+
+        with self._lock:
+            run = self._runs.get(run_id)
+            if run is None:
+                return False
+            sidecar = dict(run.sidecar)
+            run_dir = run.run_dir
+
+        sidecar.update(patch)
+        write_sidecar(run_dir, sidecar)
+
+        try:
+            new_mtime = (run_dir / SIDECAR_NAME).stat().st_mtime
+        except OSError:
+            new_mtime = None
+
+        with self._lock:
+            run = self._runs.get(run_id)
+            if run is not None:
+                run.sidecar = sidecar
+                if new_mtime is not None:
+                    run.sidecar_mtime = new_mtime
+
+        self._publish("update", {"changed": [run_id], "removed": []})
+        return True
+
     def logs_index(self, run_id: str) -> Optional[dict]:
         """Discover ``.out`` / ``.err`` / ``.log`` files for a run.
 
diff --git a/stable_pretraining/web/server.py b/stable_pretraining/web/server.py
index fe1d3b019..b3cfb5da2 100644
--- a/stable_pretraining/web/server.py
+++ b/stable_pretraining/web/server.py
@@ -10,6 +10,7 @@
     GET /api/logs?run_id=…           → JSON list of available .out/.err streams
     GET /api/log-content?run_id=…&stream_id=… → text/plain (last ~4 MiB)
     GET /api/stream                  → Server-Sent Events stream of update deltas
+    PATCH /api/run-meta              → mutate display_name / notes / tags / archived
 
 ThreadingHTTPServer spawns a thread per request; SSE handlers hold a
 thread for the connection lifetime, which is fine for a local viewer.
@@ -158,6 +159,47 @@ def do_GET(self) -> None:  # noqa: N802
         except (BrokenPipeError, ConnectionResetError):
             pass
 
+    def do_PATCH(self) -> None:  # noqa: N802
+        try:
+            url = urlparse(self.path)
+            if url.path != "/api/run-meta":
+                self.send_error(404, "Not Found")
+                return
+
+            length = int(self.headers.get("Content-Length", 0))
+            if length > 64 * 1024:
+                self._serve_json({"error": "request body too large"}, 400)
+                return
+
+            try:
+                patch = json.loads(self.rfile.read(length))
+            except (json.JSONDecodeError, ValueError):
+                self._serve_json({"error": "invalid JSON"}, 400)
+                return
+
+            if not isinstance(patch, dict):
+                self._serve_json({"error": "body must be a JSON object"}, 400)
+                return
+
+            run_id = patch.pop("run_id", None)
+            if not run_id or not isinstance(run_id, str):
+                self._serve_json({"error": "missing run_id"}, 400)
+                return
+
+            try:
+                found = self.scanner.patch_run_meta(run_id, patch)
+            except ValueError as exc:
+                self._serve_json({"error": str(exc)}, 400)
+                return
+
+            if not found:
+                self._serve_json({"error": "run not found"}, 404)
+                return
+
+            self._serve_json({"ok": True})
+        except (BrokenPipeError, ConnectionResetError):
+            pass
+
     # ---- helpers ----
 
     def _serve_asset(self, name: str, ctype: Optional[str] = None) -> None:

From f708f6236770a9533b6f36fc86be5a3f10dd7b24 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 19:27:37 -0400
Subject: [PATCH 04/31] now you can add notes per note that persist throughout,
 in addition, the text box changes in size based on the amount of notes

---
 stable_pretraining/web/assets/app.css | 21 ++++++++++
 stable_pretraining/web/assets/app.js  | 58 +++++++++++++++++++++++++--
 2 files changed, 75 insertions(+), 4 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 871329caa..33ee8d668 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -956,6 +956,27 @@ body {
 .kv-val.bool { color: var(--info); }
 .kv-val.null { color: var(--muted-2); font-style: italic; }
 
+/* ---- notes textarea ---------------------------------------------------- */
+.notes-section { margin-bottom: 6px; }
+.notes-textarea {
+  width: 100%; box-sizing: border-box;
+  background: var(--bg); color: var(--text);
+  border: 1px solid transparent; border-radius: 4px;
+  padding: 5px 7px;
+  font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
+  font-size: 12px; line-height: 1.5;
+  resize: none; overflow: hidden;
+  min-height: 32px;
+  transition: border-color 0.15s, background 0.15s;
+}
+.notes-textarea::placeholder { color: var(--muted-2); font-style: italic; }
+.notes-textarea:hover { border-color: var(--border); }
+.notes-textarea:focus {
+  outline: none;
+  border-color: var(--accent);
+  background: var(--surface);
+}
+
 /* --- initial-scan loading overlay -------------------------------------- */
 #scan-progress {
   position: fixed;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 2b3ee7e42..a6e457fd9 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -2214,12 +2214,18 @@
     document.getElementById('detail-dot').style.background = runColor(r.run_id);
     document.getElementById('detail-title').textContent = r.display_name || r.run_id;
 
+    const body = document.getElementById('detail-body');
+
+    // Reuse the notes section if the user is actively editing to avoid losing cursor.
+    const prevNotes = body.querySelector('.notes-section');
+    const editing = prevNotes && document.activeElement === prevNotes.querySelector('textarea');
+    const notesSec = editing ? prevNotes : _makeNotesSection(r);
+
     const meta = {
       run_dir: r.run_dir,
       status: r.status,
       created_at: fmtTime(r.created_at),
       tags: (r.tags || []).join(', ') || null,
-      notes: r.notes || null,
       checkpoint_path: r.checkpoint_path,
     };
 
@@ -2232,19 +2238,63 @@
     const smSec = buildKVSection('summary', r.summary || {}, filter);
     if (smSec) sections.push(smSec);
 
-    const body = document.getElementById('detail-body');
     if (!sections.length) {
       const empty = document.createElement('div');
       empty.style.color = '#4a5568';
       empty.style.padding = '16px 0';
       empty.style.textAlign = 'center';
       empty.textContent = filter ? 'no keys match the filter' : 'no config recorded';
-      body.replaceChildren(empty);
+      body.replaceChildren(notesSec, empty);
     } else {
-      body.replaceChildren(...sections);
+      body.replaceChildren(notesSec, ...sections);
+    }
+
+    // Fit textarea height to content. Deferred so the browser has laid out the
+    // element — renderDetail may be called while the modal is still hidden,
+    // in which case scrollHeight is 0 until the next paint.
+    if (!editing) {
+      const ta = notesSec.querySelector('textarea');
+      if (ta.value) {
+        requestAnimationFrame(() => {
+          ta.style.height = 'auto';
+          ta.style.height = ta.scrollHeight + 'px';
+        });
+      }
     }
   }
 
+  function _makeNotesSection(r) {
+    const sec = document.createElement('section');
+    sec.className = 'detail-section notes-section';
+    const h = document.createElement('h3');
+    h.textContent = 'notes';
+    const ta = document.createElement('textarea');
+    ta.className = 'notes-textarea';
+    ta.placeholder = 'add notes…';
+    ta.value = r.notes || '';
+    ta.addEventListener('input', () => {
+      ta.style.height = 'auto';
+      ta.style.height = ta.scrollHeight + 'px';
+    });
+    ta.addEventListener('blur', async () => {
+      const run = state.runs.get(state.detailRunId);
+      if (!run) return;
+      const newVal = ta.value;
+      const oldVal = run.notes || '';
+      if (newVal === oldVal) return;
+      run.notes = newVal;
+      try {
+        await patchRunMeta(run.run_id, { notes: newVal });
+      } catch (e) {
+        console.warn('notes patch failed', e);
+        ta.value = run.notes = oldVal;
+      }
+    });
+    sec.appendChild(h);
+    sec.appendChild(ta);
+    return sec;
+  }
+
   // ---- SSE --------------------------------------------------------------
 
   function startSSE() {

From 6009554fe173e19d59b6ed13bacd2cf1f914bcb4 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 19:40:33 -0400
Subject: [PATCH 05/31] added a small live display to logs outputting, in
 addition, automatic refreshing for those .out and .err

---
 stable_pretraining/web/assets/app.css    | 13 ++++
 stable_pretraining/web/assets/app.js     | 85 ++++++++++++++++++++++--
 stable_pretraining/web/assets/index.html |  4 ++
 3 files changed, 95 insertions(+), 7 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 33ee8d668..158523bec 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -404,6 +404,19 @@ body {
   font-style: italic;
   white-space: normal;
 }
+.log-live-badge {
+  font-size: 10px; font-weight: 600; letter-spacing: 0.4px;
+  color: var(--success, #48bb78);
+  border: 1px solid var(--success, #48bb78);
+  border-radius: 3px;
+  padding: 1px 5px;
+  text-transform: uppercase;
+  animation: live-pulse 2s ease-in-out infinite;
+}
+@keyframes live-pulse {
+  0%, 100% { opacity: 1; }
+  50%       { opacity: 0.5; }
+}
 
 /* ---- header status stats ------------------------------------------- */
 #status-row { display: flex; gap: 8px; align-items: center; font-size: 11px; color: var(--muted); }
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index a6e457fd9..0f27ec0ca 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -35,8 +35,12 @@
     // Currently selected (runId, stream_id) for each kind, persisted across
     // tab switches so the user doesn't lose their place.
     logSelection: { out: null, err: null },  // null or {runId, streamId}
+    logLivePaused: { out: false, err: false },
   };
 
+  // setInterval handles for log auto-refresh; null when not running.
+  const _logLiveTimers = { out: null, err: null };
+
   const SYNC_KEY = 'sptweb-x';
 
   // ---- theme -----------------------------------------------------------
@@ -2345,6 +2349,11 @@
       if (state.detailRunId && removed.includes(state.detailRunId)) {
         closeDetail();
       }
+      // A run may have transitioned from running → completed/failed; check
+      // whether the live timer should be stopped.
+      if (state.activeTab === 'out' || state.activeTab === 'err') {
+        _updateLiveState(state.activeTab);
+      }
     });
     es.addEventListener('error', () => {
       // Browser auto-reconnects.
@@ -2408,8 +2417,13 @@
       pane.classList.toggle('active', pane.id === `tab-${name}`);
     }
     saveState();
+    // Stop live timers on both kinds first; isLiveLog now returns false for
+    // whichever kind is no longer the active tab.
+    _updateLiveState('out');
+    _updateLiveState('err');
     if (name === 'out' || name === 'err') {
       // Lazily fetch logs the first time the user opens a log tab.
+      // renderLogTab calls _updateLiveState(kind) to restart live if applicable.
       refreshLogStreamsForVisibleRuns().then(() => renderLogTab(name));
     } else if (name === 'table') {
       renderRunsTable();
@@ -2418,6 +2432,37 @@
 
   // ---- logs -------------------------------------------------------------
 
+  function isLiveLog(kind) {
+    if (state.activeTab !== kind) return false;
+    if (state.logLivePaused[kind]) return false;
+    const sel = state.logSelection[kind];
+    if (!sel) return false;
+    const run = state.runs.get(sel.runId);
+    return !!(run && run.status === 'running');
+  }
+
+  function _updateLiveState(kind) {
+    const live = isLiveLog(kind);
+    const badgeId = kind === 'err' ? 'logs-err-live-badge' : 'logs-live-badge';
+    const pauseId = kind === 'err' ? 'logs-err-pause' : 'logs-pause';
+    const badge = document.getElementById(badgeId);
+    const pauseBtn = document.getElementById(pauseId);
+    if (badge) badge.hidden = !live;
+    if (pauseBtn) pauseBtn.hidden = !live;
+    if (live) {
+      if (_logLiveTimers[kind]) return;  // already running
+      _logLiveTimers[kind] = setInterval(() => {
+        if (!isLiveLog(kind)) { _updateLiveState(kind); return; }
+        loadLogContent(kind);
+      }, 5000);
+    } else {
+      if (_logLiveTimers[kind]) {
+        clearInterval(_logLiveTimers[kind]);
+        _logLiveTimers[kind] = null;
+      }
+    }
+  }
+
   async function fetchLogStreams(runId) {
     try {
       const data = await fetchJSON(`/api/logs?run_id=${encodeURIComponent(runId)}`);
@@ -2472,6 +2517,7 @@
         ? 'select one or more runs on the left to view their logs.'
         : `no .${kind} files were found for the selected run(s).`;
       state.logSelection[kind] = null;
+      _updateLiveState(kind);
       return;
     }
 
@@ -2490,6 +2536,7 @@
     }
     state.logSelection[kind] = { runId: active.runId, streamId: active.streamId };
     await loadLogContent(kind);
+    _updateLiveState(kind);
   }
 
   async function loadLogContent(kind) {
@@ -2501,8 +2548,14 @@
       view.textContent = '';
       return;
     }
+    // Capture scroll position before any DOM change; treat empty / first-load
+    // as "at bottom" so we scroll down on the initial fetch.
+    const hasContent = view.textContent && !view.classList.contains('empty')
+      && view.textContent !== 'loading...';
+    const atBottom = !hasContent
+      || (view.scrollHeight - view.scrollTop - view.clientHeight < 50);
     view.classList.remove('empty');
-    view.textContent = 'loading...';
+    if (!hasContent) view.textContent = 'loading...';
     try {
       const r = await fetch(
         `/api/log-content?run_id=${encodeURIComponent(sel.runId)}&stream_id=${encodeURIComponent(sel.streamId)}`,
@@ -2510,9 +2563,9 @@
       );
       if (!r.ok) throw new Error(`HTTP ${r.status}`);
       const txt = await r.text();
+      const prevScrollTop = view.scrollTop;
       view.textContent = txt || '(empty)';
-      // Auto-scroll to the bottom (most recent output).
-      view.scrollTop = view.scrollHeight;
+      view.scrollTop = atBottom ? view.scrollHeight : prevScrollTop;
     } catch (e) {
       view.classList.add('empty');
       view.textContent = `failed to load: ${e.message || e}`;
@@ -2608,18 +2661,36 @@
     if (outSel) outSel.addEventListener('change', () => {
       const [runId, streamId] = outSel.value.split('::');
       state.logSelection.out = { runId, streamId };
-      loadLogContent('out');
+      state.logLivePaused.out = false;
+      loadLogContent('out').then(() => _updateLiveState('out'));
     });
     const errSel = document.getElementById('logs-err-stream-selector');
     if (errSel) errSel.addEventListener('change', () => {
       const [runId, streamId] = errSel.value.split('::');
       state.logSelection.err = { runId, streamId };
-      loadLogContent('err');
+      state.logLivePaused.err = false;
+      loadLogContent('err').then(() => _updateLiveState('err'));
     });
     const outRefresh = document.getElementById('logs-refresh');
-    if (outRefresh) outRefresh.addEventListener('click', () => loadLogContent('out'));
+    if (outRefresh) outRefresh.addEventListener('click', () => {
+      state.logLivePaused.out = false;
+      loadLogContent('out').then(() => _updateLiveState('out'));
+    });
     const errRefresh = document.getElementById('logs-err-refresh');
-    if (errRefresh) errRefresh.addEventListener('click', () => loadLogContent('err'));
+    if (errRefresh) errRefresh.addEventListener('click', () => {
+      state.logLivePaused.err = false;
+      loadLogContent('err').then(() => _updateLiveState('err'));
+    });
+    const outPause = document.getElementById('logs-pause');
+    if (outPause) outPause.addEventListener('click', () => {
+      state.logLivePaused.out = true;
+      _updateLiveState('out');
+    });
+    const errPause = document.getElementById('logs-err-pause');
+    if (errPause) errPause.addEventListener('click', () => {
+      state.logLivePaused.err = true;
+      _updateLiveState('err');
+    });
 
     document.getElementById('theme-toggle').addEventListener('click', () => {
       applyTheme(state.theme === 'dark' ? 'light' : 'dark');
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 86e96f5fe..111f3168d 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -78,6 +78,8 @@
         <label class="control">stream
           <select id="logs-stream-selector"></select>
         </label>
+        <span id="logs-live-badge" class="log-live-badge" hidden>live</span>
+        <button id="logs-pause" class="icon-btn" type="button" title="pause auto-refresh" hidden>⏸</button>
         <button id="logs-refresh" class="icon-btn" type="button" title="refresh">↻</button>
       </div>
       <pre id="logs-out" class="log-view"></pre>
@@ -87,6 +89,8 @@
         <label class="control">stream
           <select id="logs-err-stream-selector"></select>
         </label>
+        <span id="logs-err-live-badge" class="log-live-badge" hidden>live</span>
+        <button id="logs-err-pause" class="icon-btn" type="button" title="pause auto-refresh" hidden>⏸</button>
         <button id="logs-err-refresh" class="icon-btn" type="button" title="refresh">↻</button>
       </div>
       <pre id="logs-err" class="log-view"></pre>

From 5bcbfc13c80df43b984141cd1f00bb0967200447 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 20:00:20 -0400
Subject: [PATCH 06/31] added duration for the runs

---
 stable_pretraining/registry/_sidecar.py |  2 +
 stable_pretraining/registry/logger.py   |  3 ++
 stable_pretraining/web/assets/app.css   |  5 +++
 stable_pretraining/web/assets/app.js    | 55 ++++++++++++++++++++++---
 stable_pretraining/web/scan.py          |  8 +++-
 5 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/stable_pretraining/registry/_sidecar.py b/stable_pretraining/registry/_sidecar.py
index 06c8da85e..b5b902a49 100644
--- a/stable_pretraining/registry/_sidecar.py
+++ b/stable_pretraining/registry/_sidecar.py
@@ -56,6 +56,7 @@ def make_sidecar(
     run_dir: str,
     status: str = "running",
     created_at: Optional[float] = None,
+    ended_at: Optional[float] = None,
     hparams: Optional[Dict[str, Any]] = None,
     summary: Optional[Dict[str, Any]] = None,
     tags: Optional[List[str]] = None,
@@ -70,6 +71,7 @@ def make_sidecar(
         "run_dir": run_dir,
         "status": status,
         "created_at": created_at if created_at is not None else now,
+        "ended_at": ended_at,
         "updated_at": now,
         "tags": list(tags or []),
         "notes": notes or "",
diff --git a/stable_pretraining/registry/logger.py b/stable_pretraining/registry/logger.py
index d952e63fe..43e098ad1 100644
--- a/stable_pretraining/registry/logger.py
+++ b/stable_pretraining/registry/logger.py
@@ -177,6 +177,7 @@ def __init__(
         # the registry can order runs chronologically regardless of how
         # often we flush.
         self._created_at: Optional[float] = None
+        self._ended_at: Optional[float] = None
         # First-write flag for summary.json — used to log a one-shot info
         # line on creation, then debug lines on subsequent rewrites so we
         # don't spam every flush.
@@ -262,6 +263,7 @@ def save(self) -> None:
     def finalize(self, status: str) -> None:
         # Map Lightning status strings to our canonical vocabulary.
         self._status = {"success": "completed", "failed": "failed"}.get(status, status)
+        self._ended_at = time.time()
         # Parent writes CSVs.  We don't call super().finalize first
         # because _experiment may be None on rank-zero callers that
         # never logged — super() handles that no-op correctly.
@@ -406,6 +408,7 @@ def _write_sidecar(self) -> None:
             run_dir=str(self._run_dir),
             status=self._status,
             created_at=self._created_at,
+            ended_at=self._ended_at,
             hparams=self._hparams,
             summary=self._summary,
             tags=self._tags,
diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 158523bec..5d78a1b0c 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -284,6 +284,11 @@ body {
 .run-status.completed { color: var(--good); }
 .run-status.failed    { color: var(--bad); }
 .run-status.running   { color: var(--accent); }
+.run-dur {
+  font-size: 10px; color: var(--muted-2);
+  font-variant-numeric: tabular-nums;
+  white-space: nowrap; flex-shrink: 0;
+}
 
 .run-info {
   background: none; border: none; cursor: pointer;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 0f27ec0ca..d5878a2e7 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -737,6 +737,14 @@
       row.appendChild(st);
     }
 
+    if (r.status === 'running' || r.ended_at) {
+      const dur = document.createElement('div');
+      dur.className = 'run-dur';
+      dur.dataset.runId = r.run_id;
+      dur.textContent = fmtRunDur(r) || '';
+      row.appendChild(dur);
+    }
+
     const info = document.createElement('button');
     info.type = 'button';
     info.className = 'run-info';
@@ -2140,6 +2148,28 @@
     return d.toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC');
   }
 
+  function fmtDuration(secs) {
+    if (secs == null || secs < 0) return null;
+    const s = Math.floor(secs);
+    if (s < 60) return '<1m';
+    const m = Math.floor(s / 60);
+    if (m < 60) return `${m}m`;
+    const h = Math.floor(m / 60);
+    const rm = m % 60;
+    if (h < 24) return rm > 0 ? `${h}h ${rm}m` : `${h}h`;
+    const d = Math.floor(h / 24);
+    const rh = h % 24;
+    return rh > 0 ? `${d}d ${rh}h` : `${d}d`;
+  }
+
+  function fmtRunDur(r) {
+    if (!r.created_at) return null;
+    const now = Date.now() / 1000;
+    if (r.status === 'running') return fmtDuration(now - r.created_at);
+    if (r.ended_at) return fmtDuration(r.ended_at - r.created_at);
+    return null;
+  }
+
   function classifyValue(v) {
     if (v == null) return 'null';
     if (typeof v === 'number') return 'num';
@@ -2179,10 +2209,10 @@
     }
   }
 
-  function buildKVSection(label, kv, filter) {
+  function buildKVSection(label, kv, filter, sorted = true) {
     const entries = Object.entries(kv)
-      .filter(([k]) => !filter || k.toLowerCase().includes(filter))
-      .sort((a, b) => a[0].localeCompare(b[0]));
+      .filter(([k]) => !filter || k.toLowerCase().includes(filter));
+    if (sorted) entries.sort((a, b) => a[0].localeCompare(b[0]));
     if (!entries.length) return null;
 
     const section = document.createElement('section');
@@ -2225,17 +2255,25 @@
     const editing = prevNotes && document.activeElement === prevNotes.querySelector('textarea');
     const notesSec = editing ? prevNotes : _makeNotesSection(r);
 
+    const now = Date.now() / 1000;
+    let durSecs = null;
+    if (r.created_at) {
+      if (r.ended_at) durSecs = r.ended_at - r.created_at;
+      else if (r.status === 'running') durSecs = now - r.created_at;
+    }
     const meta = {
       run_dir: r.run_dir,
       status: r.status,
-      created_at: fmtTime(r.created_at),
+      started: fmtTime(r.created_at),
+      ended: r.ended_at ? fmtTime(r.ended_at) : null,
+      duration: fmtDuration(durSecs),
       tags: (r.tags || []).join(', ') || null,
       checkpoint_path: r.checkpoint_path,
     };
 
     const filter = state.detailFilter.toLowerCase();
     const sections = [];
-    const metaSec = buildKVSection('meta', meta, filter);
+    const metaSec = buildKVSection('meta', meta, filter, false);
     if (metaSec) sections.push(metaSec);
     const hpSec = buildKVSection('hparams', r.hparams || {}, filter);
     if (hpSec) sections.push(hpSec);
@@ -2777,6 +2815,13 @@
     wireControls();
     syncControlsToState();
     initTheme();
+    // Tick every minute to keep duration displays current for running runs.
+    setInterval(() => {
+      for (const el of document.querySelectorAll('.run-dur[data-run-id]')) {
+        const r = state.runs.get(el.dataset.runId);
+        if (r) el.textContent = fmtRunDur(r) || '';
+      }
+    }, 60_000);
     // Start SSE first so we don't miss progress events emitted between the
     // status fetch and the first scan-tick.
     startSSE();
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index 8b1bd8c98..088eae289 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import Any, Optional
 
-from ..registry._sidecar import SIDECAR_NAME, write_sidecar
+from ..registry._sidecar import SIDECAR_NAME, heartbeat_mtime, write_sidecar
 
 
 @dataclass
@@ -397,12 +397,18 @@ def _serialize(run: _Run) -> dict:
             or run.run_id.rsplit("/", 1)[-1]
             or run.run_id
         )
+        try:
+            hb_at = heartbeat_mtime(run.run_dir)
+        except OSError:
+            hb_at = None
         return {
             "run_id": run.run_id,
             "run_dir": str(run.run_dir),
             "display_name": display_name,
             "status": s.get("status"),
             "created_at": s.get("created_at"),
+            "ended_at": s.get("ended_at"),
+            "heartbeat_at": hb_at,
             "tags": s.get("tags") or [],
             "notes": s.get("notes") or "",
             "hparams": s.get("hparams") or {},

From ae86b50df074de1cafb988f3095aa25c99fcb140 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 4 Jun 2026 20:22:40 -0400
Subject: [PATCH 07/31] added functionality to detect stale runs, that would be
 shown as running but are not actually

---
 stable_pretraining/web/assets/app.css    |  9 +++
 stable_pretraining/web/assets/app.js     | 90 ++++++++++++++++++------
 stable_pretraining/web/assets/index.html |  1 +
 3 files changed, 77 insertions(+), 23 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 5d78a1b0c..63e280fe0 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -284,11 +284,19 @@ body {
 .run-status.completed { color: var(--good); }
 .run-status.failed    { color: var(--bad); }
 .run-status.running   { color: var(--accent); }
+.run-status.stale     { color: var(--warn, #f59e0b); }
 .run-dur {
   font-size: 10px; color: var(--muted-2);
   font-variant-numeric: tabular-nums;
   white-space: nowrap; flex-shrink: 0;
 }
+.run-dot.stale {
+  background: none; border: none;
+  color: var(--warn, #f59e0b);
+  font-size: 10px; line-height: 1;
+  display: flex; align-items: center; justify-content: center;
+  cursor: help;
+}
 
 .run-info {
   background: none; border: none; cursor: pointer;
@@ -433,6 +441,7 @@ body {
 .status-stat .dot.running   { background: #22d3ee; }
 .status-stat .dot.completed { background: #34d399; }
 .status-stat .dot.failed    { background: #f87171; }
+.status-stat .dot.stale     { background: var(--warn, #f59e0b); }
 .status-stat .n { color: var(--text); font-weight: 600; font-variant-numeric: tabular-nums; }
 
 /* ---- main / charts -------------------------------------------------- */
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index d5878a2e7..2b5febffc 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -436,6 +436,7 @@
 
   function valueAt(run, key) {
     if (!key) return null;
+    if (key === 'status') return effectiveStatus(run);
     if (key.indexOf('.') < 0) return run[key];
     const dot = key.indexOf('.');
     const ns = key.slice(0, dot);
@@ -671,8 +672,15 @@
     row.dataset.runId = r.run_id;
 
     const dot = document.createElement('div');
-    dot.className = 'run-dot';
-    dot.style.background = runColor(r.run_id);
+    if (isStale(r)) {
+      dot.className = 'run-dot stale';
+      dot.textContent = '⚠';
+      const staleMins = Math.floor((Date.now() / 1000 - r.heartbeat_at) / 60);
+      dot.title = `no heartbeat for ${staleMins > 0 ? staleMins + 'm' : '<1m'} — may have crashed`;
+    } else {
+      dot.className = 'run-dot';
+      dot.style.background = runColor(r.run_id);
+    }
     row.appendChild(dot);
 
     // Name column: primary display_name + optional dimmed run_id hint.
@@ -732,8 +740,9 @@
 
     if (r.status) {
       const st = document.createElement('div');
-      st.className = `run-status ${r.status}`;
-      st.textContent = r.status;
+      const es = effectiveStatus(r);
+      st.className = `run-status ${es}`;
+      st.textContent = es;
       row.appendChild(st);
     }
 
@@ -780,8 +789,8 @@
     // that affect appearance (visibility, loading-pulse) are repainted by
     // touching only the affected row's classList below.
     const layoutKey = JSON.stringify({
-      groupKeys: groups.map(g => [g.key, g.runs.map(r => [r.run_id, r.display_name])]),
-      ungrouped: ungrouped.map(r => [r.run_id, r.display_name]),
+      groupKeys: groups.map(g => [g.key, g.runs.map(r => [r.run_id, r.display_name, isStale(r)])]),
+      ungrouped: ungrouped.map(r => [r.run_id, r.display_name, isStale(r)]),
       open: [...state.openSidebarGroups].sort(),
     });
     if (state._lastListKey === layoutKey) {
@@ -1864,9 +1873,10 @@
     wrap.className = 'status-bars';
     const colors = {
       completed: '#34d399', running: '#22d3ee',
+      stale: '#f59e0b',
       failed: '#f87171', unknown: '#6b7280',
     };
-    const order = ['completed', 'running', 'failed'];
+    const order = ['completed', 'running', 'stale', 'failed'];
     const seen = new Set(Object.keys(counts));
     const sorted = [
       ...order.filter(k => seen.has(k)),
@@ -1946,8 +1956,15 @@
     const row = document.createElement('div');
     row.className = 'recent-run';
     const dot = document.createElement('div');
-    dot.className = 'run-dot';
-    dot.style.background = runColor(r.run_id);
+    if (isStale(r)) {
+      dot.className = 'run-dot stale';
+      dot.textContent = '⚠';
+      const staleMins = Math.floor((Date.now() / 1000 - r.heartbeat_at) / 60);
+      dot.title = `no heartbeat for ${staleMins > 0 ? staleMins + 'm' : '<1m'} — may have crashed`;
+    } else {
+      dot.className = 'run-dot';
+      dot.style.background = runColor(r.run_id);
+    }
     row.appendChild(dot);
     const name = document.createElement('div');
     name.className = 'recent-run-name';
@@ -1956,8 +1973,9 @@
     row.appendChild(name);
     if (r.status) {
       const st = document.createElement('div');
-      st.className = `run-status ${r.status}`;
-      st.textContent = r.status;
+      const es = effectiveStatus(r);
+      st.className = `run-status ${es}`;
+      st.textContent = es;
       row.appendChild(st);
     }
     const ago = document.createElement('div');
@@ -1975,13 +1993,16 @@
     const days = 30;
     const startDay = Math.floor((now - (days - 1) * oneDay) / oneDay) * oneDay;
     const totalBins = new Array(days).fill(0);
-    const failBins = new Array(days).fill(0);
+    const failBins  = new Array(days).fill(0);
+    const staleBins = new Array(days).fill(0);
     for (const r of runs) {
       if (!r.created_at) continue;
       const idx = Math.floor((r.created_at - startDay) / oneDay);
       if (idx < 0 || idx >= days) continue;
       totalBins[idx]++;
-      if (r.status === 'failed') failBins[idx]++;
+      const es = effectiveStatus(r);
+      if (es === 'failed') failBins[idx]++;
+      if (es === 'stale')  staleBins[idx]++;
     }
     const xs = new Array(days);
     for (let i = 0; i < days; i++) xs[i] = startDay + i * oneDay + oneDay / 2;
@@ -1990,10 +2011,11 @@
     const bars = uPlot.paths.bars
       ? uPlot.paths.bars({ size: [0.7, 80], align: 0 })
       : undefined;
-    const muted = themeColor('muted') || '#8aa0b8';
-    const grid = themeColor('grid') || '#1f2630';
+    const muted  = themeColor('muted')  || '#8aa0b8';
+    const grid   = themeColor('grid')   || '#1f2630';
     const accent = themeColor('accent') || '#22d3ee';
-    const bad = themeColor('bad') || '#f87171';
+    const bad    = themeColor('bad')    || '#f87171';
+    const warn   = themeColor('warn')   || '#f59e0b';
     new uPlot({
       width, height: 180,
       cursor: { drag: { x: false, y: false } },
@@ -2030,20 +2052,27 @@
         {
           label: 'failed',
           stroke: bad,
-          fill: bad + '8c',    // ~55% alpha
+          fill: bad + '8c',
+          paths: bars,
+          points: { show: false },
+        },
+        {
+          label: 'stale',
+          stroke: warn,
+          fill: warn + '8c',
           paths: bars,
           points: { show: false },
         },
       ],
       legend: { show: true, live: false },
-    }, [xs, totalBins, failBins], parent);
+    }, [xs, totalBins, failBins, staleBins], parent);
   }
 
   function renderOverview(root) {
     const allRuns = [...state.runs.values()].filter(passesFilters);
     const counts = {};
     for (const r of allRuns) {
-      const s = r.status || 'unknown';
+      const s = effectiveStatus(r) || 'unknown';
       counts[s] = (counts[s] || 0) + 1;
     }
 
@@ -2065,6 +2094,7 @@
     cards.className = 'stat-cards';
     cards.appendChild(statCard('total runs', allRuns.length, '#f1f5f9'));
     cards.appendChild(statCard('running',   counts.running   || 0, '#22d3ee'));
+    if (counts.stale) cards.appendChild(statCard('stale', counts.stale, '#f59e0b'));
     cards.appendChild(statCard('completed', counts.completed || 0, '#34d399'));
     cards.appendChild(statCard('failed',    counts.failed    || 0, '#f87171'));
     wrap.appendChild(cards);
@@ -2170,6 +2200,16 @@
     return null;
   }
 
+  function isStale(r) {
+    if (r.status !== 'running') return false;
+    if (!r.heartbeat_at) return false;
+    return (Date.now() / 1000) - r.heartbeat_at > 300;
+  }
+
+  function effectiveStatus(r) {
+    return isStale(r) ? 'stale' : (r.status || null);
+  }
+
   function classifyValue(v) {
     if (v == null) return 'null';
     if (typeof v === 'number') return 'num';
@@ -2263,7 +2303,7 @@
     }
     const meta = {
       run_dir: r.run_dir,
-      status: r.status,
+      status: effectiveStatus(r),
       started: fmtTime(r.created_at),
       ended: r.ended_at ? fmtTime(r.ended_at) : null,
       duration: fmtDuration(durSecs),
@@ -2432,15 +2472,17 @@
   // ---- header stats -----------------------------------------------------
 
   function updateHeaderStats() {
-    const counts = { running: 0, completed: 0, failed: 0 };
+    const counts = { running: 0, completed: 0, failed: 0, stale: 0 };
     for (const r of state.runs.values()) {
-      const s = r.status || 'unknown';
+      const s = effectiveStatus(r) || 'unknown';
       if (s in counts) counts[s] += 1;
     }
     for (const [k, v] of Object.entries(counts)) {
       const el = document.querySelector(`#stat-${k} .n`);
       if (el) el.textContent = String(v);
     }
+    const staleChip = document.getElementById('stat-stale');
+    if (staleChip) staleChip.hidden = counts.stale === 0;
   }
 
   // ---- tabs -------------------------------------------------------------
@@ -2815,12 +2857,14 @@
     wireControls();
     syncControlsToState();
     initTheme();
-    // Tick every minute to keep duration displays current for running runs.
+    // Tick every minute to keep duration displays and stale indicators current.
     setInterval(() => {
       for (const el of document.querySelectorAll('.run-dur[data-run-id]')) {
         const r = state.runs.get(el.dataset.runId);
         if (r) el.textContent = fmtRunDur(r) || '';
       }
+      renderRunList();
+      updateHeaderStats();
     }, 60_000);
     // Start SSE first so we don't miss progress events emitted between the
     // status fetch and the first scan-tick.
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 111f3168d..2eeb0b3f6 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -16,6 +16,7 @@
       <span class="status-stat" id="stat-running"   title="running runs">·  <span class="dot running"></span><span class="n">0</span> running</span>
       <span class="status-stat" id="stat-completed" title="completed runs">·  <span class="dot completed"></span><span class="n">0</span> done</span>
       <span class="status-stat" id="stat-failed"    title="failed runs">·  <span class="dot failed"></span><span class="n">0</span> failed</span>
+      <span class="status-stat" id="stat-stale"     title="stale runs — no heartbeat for >5m" hidden>·  <span class="dot stale"></span><span class="n">0</span> stale</span>
     </div>
     <div class="spacer"></div>
     <button id="theme-toggle" class="icon-btn" type="button" title="toggle light/dark theme" aria-label="toggle theme"></button>

From b40300fc69518e1a6121078e27b7d4fe4fad01ce Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 08:54:53 -0400
Subject: [PATCH 08/31] added scatterplot to compare different metrics across
 runs

---
 stable_pretraining/web/assets/app.css |  39 +++++
 stable_pretraining/web/assets/app.js  | 205 ++++++++++++++++++++++++++
 2 files changed, 244 insertions(+)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 63e280fe0..2aac23261 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -667,6 +667,45 @@ body {
 .annot-val { text-align: right; white-space: nowrap; }
 .annot-best-bold { font-weight: 700; color: var(--text-strong); }
 
+/* ---- scatter section -------------------------------------------------- */
+
+.scatter-section {
+  margin: 16px 12px 8px;
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  overflow: hidden;
+}
+.scatter-header {
+  display: flex; align-items: center; gap: 16px; flex-wrap: wrap;
+  padding: 8px 12px;
+  border-bottom: 1px solid var(--border);
+}
+.scatter-title {
+  font-size: 11px; font-weight: 600;
+  color: var(--muted); text-transform: uppercase; letter-spacing: 0.6px;
+  margin: 0; white-space: nowrap;
+}
+.scatter-controls {
+  display: flex; align-items: center; gap: 12px; flex-wrap: wrap;
+}
+.scatter-axis-label {
+  font-size: 11px; color: var(--muted);
+  display: flex; align-items: center; gap: 4px;
+}
+.scatter-axis-sel {
+  background: var(--bg); color: var(--text);
+  border: 1px solid var(--border); border-radius: 4px;
+  padding: 2px 5px; font-size: 11px; font-family: inherit;
+}
+.scatter-axis-sel:focus { outline: none; border-color: var(--accent); }
+.scatter-plot { padding: 8px; }
+.scatter-empty {
+  padding: 24px 12px;
+  color: var(--muted); font-size: 12px;
+  text-align: center; font-style: italic;
+}
+
 /* ---- runs table ------------------------------------------------------- */
 
 #table-controls {
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 2b5febffc..fab2bec85 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -36,11 +36,16 @@
     // tab switches so the user doesn't lose their place.
     logSelection: { out: null, err: null },  // null or {runId, streamId}
     logLivePaused: { out: false, err: false },
+    scatterX: null,   // 'hparams.lr' | 'summary.val_acc' | …
+    scatterY: null,
   };
 
   // setInterval handles for log auto-refresh; null when not running.
   const _logLiveTimers = { out: null, err: null };
 
+  // Active uPlot instance for the scatter panel; null when not shown.
+  let _scatterPlot = null;
+
   const SYNC_KEY = 'sptweb-x';
 
   // ---- theme -----------------------------------------------------------
@@ -99,6 +104,8 @@
       smoothing: state.smoothing,
       metricSearch: state.metricSearch,
       tableColSearch: state.tableColSearch,
+      scatterX: state.scatterX,
+      scatterY: state.scatterY,
       activeTab: state.activeTab,
       theme: state.theme,
     };
@@ -142,6 +149,8 @@
       if (typeof saved.smoothing === 'number') state.smoothing = saved.smoothing;
       if (typeof saved.metricSearch === 'string') state.metricSearch = saved.metricSearch;
       if (typeof saved.tableColSearch === 'string') state.tableColSearch = saved.tableColSearch;
+      if (typeof saved.scatterX === 'string') state.scatterX = saved.scatterX;
+      if (typeof saved.scatterY === 'string') state.scatterY = saved.scatterY;
       if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
       if (typeof saved.theme === 'string') state.theme = saved.theme;
       if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
@@ -1051,6 +1060,7 @@
       for (const { panel } of state.mediaPanels.values()) panel.remove();
       state.mediaPanels.clear();
       state._lastTreeKey = null; // force rebuild next time we leave overview
+      if (_scatterPlot) { _scatterPlot.destroy(); _scatterPlot = null; }
       renderOverview(root);
       return;
     }
@@ -1102,6 +1112,7 @@
 
     for (const name of metrics) updateChart(name);
     for (const tag of mediaTags.keys()) updateMediaPanel(tag);
+    updateScatterSection(root);
   }
 
   // ---- runs table ---------------------------------------------------------
@@ -2068,6 +2079,200 @@
     }, [xs, totalBins, failBins, staleBins], parent);
   }
 
+  // ---- scatter plot -------------------------------------------------------
+
+  function scatterNumericKeys() {
+    const keys = new Set();
+    for (const id of effectivelyVisible()) {
+      const r = state.runs.get(id);
+      if (!r) continue;
+      for (const [k, v] of Object.entries(r.summary || {}))
+        if (typeof v === 'number' && isFinite(v)) keys.add('summary.' + k);
+      for (const [k, v] of Object.entries(r.hparams || {}))
+        if (typeof v === 'number' && isFinite(v)) keys.add('hparams.' + k);
+    }
+    return [...keys].sort();
+  }
+
+  function getScatterVal(r, key) {
+    const v = valueAt(r, key);
+    return (typeof v === 'number' && isFinite(v)) ? v : null;
+  }
+
+  function _fillScatterSel(sel, keys, currentVal) {
+    sel.innerHTML = '';
+    const empty = document.createElement('option');
+    empty.value = ''; empty.textContent = '— select —';
+    sel.appendChild(empty);
+    for (const k of keys) {
+      const opt = document.createElement('option');
+      opt.value = k;
+      opt.textContent = k.replace(/^(summary|hparams)\./, (_, ns) => ns + ': ');
+      if (k === currentVal) opt.selected = true;
+      sel.appendChild(opt);
+    }
+    if (!currentVal || !keys.includes(currentVal)) sel.value = '';
+  }
+
+  function _rebuildScatterPlot(section) {
+    const plotDiv = section.querySelector('.scatter-plot');
+    if (_scatterPlot) { _scatterPlot.destroy(); _scatterPlot = null; }
+    plotDiv.replaceChildren();
+
+    const xKey = state.scatterX;
+    const yKey = state.scatterY;
+    if (!xKey || !yKey) {
+      const msg = document.createElement('div');
+      msg.className = 'scatter-empty';
+      msg.textContent = 'select x and y axes above to draw the scatter plot';
+      plotDiv.appendChild(msg);
+      return;
+    }
+
+    const validRuns = effectivelyVisible()
+      .map(id => state.runs.get(id))
+      .filter(r => r && getScatterVal(r, xKey) != null && getScatterVal(r, yKey) != null);
+
+    if (validRuns.length === 0) {
+      const msg = document.createElement('div');
+      msg.className = 'scatter-empty';
+      msg.textContent = 'no visible runs have both selected fields';
+      plotDiv.appendChild(msg);
+      return;
+    }
+
+    // Shared x-axis: deduplicated, sorted numeric x-values across all runs.
+    const allXVals = [...new Set(validRuns.map(r => getScatterVal(r, xKey)))].sort((a, b) => a - b);
+
+    const muted = themeColor('muted') || '#8aa0b8';
+    const grid  = themeColor('grid')  || '#1f2630';
+
+    const series = [{ label: '' }];
+    const data   = [allXVals];
+
+    for (const r of validRuns) {
+      const xv  = getScatterVal(r, xKey);
+      const yv  = getScatterVal(r, yKey);
+      const xi  = allXVals.indexOf(xv);
+      const yArr = new Array(allXVals.length).fill(null);
+      yArr[xi] = yv;
+      const color = runColor(r.run_id);
+      series.push({
+        label: r.display_name || r.run_id,
+        stroke: color, fill: color,
+        paths: () => null,
+        points: { show: true, size: 8, fill: color, stroke: color },
+      });
+      data.push(yArr);
+    }
+
+    const width = plotDiv.clientWidth || 600;
+    _scatterPlot = new uPlot({
+      width, height: 260,
+      cursor: { drag: { x: false, y: false } },
+      scales: { x: { time: false }, y: {} },
+      axes: [
+        {
+          label: xKey.replace(/^(summary|hparams)\./, ''),
+          stroke: muted, grid: { stroke: grid, width: 1 }, ticks: { stroke: grid },
+          size: 50,
+        },
+        {
+          label: yKey.replace(/^(summary|hparams)\./, ''),
+          stroke: muted, grid: { stroke: grid, width: 1 }, ticks: { stroke: grid },
+          size: 60,
+        },
+      ],
+      series,
+      legend: { show: true, live: true },
+    }, data, plotDiv);
+  }
+
+  function updateScatterSection(root) {
+    const visIds = effectivelyVisible();
+
+    // Hide when fewer than 2 runs are visible.
+    if (visIds.length < 2) {
+      const existing = root.querySelector('.scatter-section');
+      if (existing) {
+        if (_scatterPlot) { _scatterPlot.destroy(); _scatterPlot = null; }
+        existing.remove();
+      }
+      return;
+    }
+
+    // Create section on first use.
+    let section = root.querySelector('.scatter-section');
+    if (!section) {
+      section = document.createElement('div');
+      section.className = 'scatter-section';
+
+      const hdr = document.createElement('div');
+      hdr.className = 'scatter-header';
+      const title = document.createElement('h3');
+      title.className = 'scatter-title';
+      title.textContent = 'scatter';
+      hdr.appendChild(title);
+
+      const controls = document.createElement('div');
+      controls.className = 'scatter-controls';
+
+      const makeAxisLabel = (axis, selId) => {
+        const lbl = document.createElement('label');
+        lbl.className = 'scatter-axis-label';
+        lbl.textContent = axis + ' ';
+        const sel = document.createElement('select');
+        sel.className = 'scatter-axis-sel';
+        sel.id = selId;
+        lbl.appendChild(sel);
+        return lbl;
+      };
+
+      controls.appendChild(makeAxisLabel('x', 'scatter-x-sel'));
+      controls.appendChild(makeAxisLabel('y', 'scatter-y-sel'));
+      hdr.appendChild(controls);
+      section.appendChild(hdr);
+
+      const plotDiv = document.createElement('div');
+      plotDiv.className = 'scatter-plot';
+      section.appendChild(plotDiv);
+      root.appendChild(section);
+
+      section.querySelector('#scatter-x-sel').addEventListener('change', e => {
+        state.scatterX = e.target.value || null;
+        saveState();
+        _rebuildScatterPlot(section);
+      });
+      section.querySelector('#scatter-y-sel').addEventListener('change', e => {
+        state.scatterY = e.target.value || null;
+        saveState();
+        _rebuildScatterPlot(section);
+      });
+    }
+
+    const keys = scatterNumericKeys();
+    const xSel = section.querySelector('#scatter-x-sel');
+    const ySel = section.querySelector('#scatter-y-sel');
+    _fillScatterSel(xSel, keys, state.scatterX);
+    _fillScatterSel(ySel, keys, state.scatterY);
+
+    // Auto-select defaults on first visit: hparam for X, summary for Y.
+    if (!state.scatterX || !keys.includes(state.scatterX)) {
+      const def = keys.find(k => k.startsWith('hparams.')) || keys[0] || null;
+      state.scatterX = def;
+      if (xSel && def) xSel.value = def;
+    }
+    if (!state.scatterY || !keys.includes(state.scatterY)) {
+      const def = keys.find(k => k.startsWith('summary.'))
+        || keys.find(k => k !== state.scatterX)
+        || null;
+      state.scatterY = def;
+      if (ySel && def) ySel.value = def;
+    }
+
+    _rebuildScatterPlot(section);
+  }
+
   function renderOverview(root) {
     const allRuns = [...state.runs.values()].filter(passesFilters);
     const counts = {};

From fc11d275237ea443ea6e85e3bc0aa598205e0d84 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 09:00:45 -0400
Subject: [PATCH 09/31] csv download functionality added

---
 stable_pretraining/web/assets/app.css    |  6 ++++
 stable_pretraining/web/assets/app.js     | 38 ++++++++++++++++++++++++
 stable_pretraining/web/assets/index.html |  1 +
 3 files changed, 45 insertions(+)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 2aac23261..e3fe33343 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -393,6 +393,12 @@ body {
   margin-left: -8px;
 }
 
+#export-csv {
+  margin-left: auto;
+  font-size: 12px;
+  padding: 3px 8px;
+}
+
 .logs-controls {
   display: flex; gap: 12px; align-items: center;
   padding: 8px 12px;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index fab2bec85..5f567968d 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -2857,6 +2857,42 @@
     }
   }
 
+  function exportMetricsCSV() {
+    const visIds = effectivelyVisible();
+    const metricNames = visibleMetricNames();
+    if (!visIds.length || !metricNames.length) return;
+
+    const rows = ['run_id,run_name,metric,step,epoch,value'];
+    for (const id of visIds) {
+      const run = state.runs.get(id);
+      const name = (run && run.display_name) || id;
+      const m = state.metrics.get(id);
+      if (!m) continue;
+      for (const metric of metricNames) {
+        const col = m.metrics[metric];
+        if (!col) continue;
+        for (let i = 0; i < col.y.length; i++) {
+          const step  = col.step[i]  ?? '';
+          const epoch = col.epoch[i] ?? '';
+          const val   = col.y[i]     ?? '';
+          const escapedName   = name.includes(',')   ? `"${name.replace(/"/g, '""')}"` : name;
+          const escapedMetric = metric.includes(',') ? `"${metric.replace(/"/g, '""')}"` : metric;
+          rows.push(`${id},${escapedName},${escapedMetric},${step},${epoch},${val}`);
+        }
+      }
+    }
+
+    const blob = new Blob([rows.join('\n')], { type: 'text/csv' });
+    const url  = URL.createObjectURL(blob);
+    const a    = document.createElement('a');
+    a.href     = url;
+    a.download = 'spt_metrics.csv';
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+  }
+
   function wireControls() {
     document.getElementById('run-search').addEventListener(
       'input',
@@ -2887,6 +2923,8 @@
     document.getElementById('select-all').addEventListener('click', () => setAllVisible(true));
     document.getElementById('clear-all').addEventListener('click', () => setAllVisible(false));
 
+    document.getElementById('export-csv').addEventListener('click', exportMetricsCSV);
+
     document.getElementById('add-filter-btn').addEventListener('click', () => openFilterDraft(null));
 
     document.getElementById('group-by').addEventListener('change', e => {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 2eeb0b3f6..0e5611044 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -71,6 +71,7 @@
         <input id="metric-search" type="search" placeholder="filter metrics by name..."
                autocomplete="off" spellcheck="false">
         <span id="metric-search-status" class="metric-search-status"></span>
+        <button id="export-csv" class="icon-btn" type="button" title="export visible metrics as CSV">⬇ CSV</button>
       </div>
       <div id="charts"><div class="empty-state">select a run on the left to plot its metrics</div></div>
     </div>

From e235294ce59e9a3e8c3f30a804eb79dbaee25d59 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 09:16:24 -0400
Subject: [PATCH 10/31] added resizing functionality for the tables allowing to
 easily reset zooming in

---
 stable_pretraining/web/assets/app.css | 18 +++++++++++++++
 stable_pretraining/web/assets/app.js  | 32 +++++++++++++++++++++++++--
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index e3fe33343..31e5edc86 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -630,6 +630,19 @@ body {
   overflow: hidden; white-space: nowrap; text-overflow: ellipsis;
 }
 .chart-body { width: 100%; }
+.chart-zoom-reset {
+  flex-shrink: 0;
+  margin-left: auto;
+  background: none;
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  color: var(--muted);
+  font-size: 13px;
+  line-height: 1;
+  padding: 1px 5px;
+  cursor: pointer;
+}
+.chart-zoom-reset:hover { color: var(--text-strong); border-color: var(--accent); }
 
 /* ---- chart annotations ------------------------------------------------ */
 .chart-annot {
@@ -897,6 +910,11 @@ body {
 
 /* uPlot themed */
 .u-cursor-pt { mix-blend-mode: normal; }
+.u-select {
+  background: var(--accent-bg-strong);
+  border: 1px solid var(--accent);
+  border-radius: 2px;
+}
 
 /* In-chart cursor tooltip (custom uPlot plugin) */
 .spt-tip {
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 5f567968d..c6d840c3f 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -1295,6 +1295,13 @@
     span.textContent = displayName || fullName;
     span.title = fullName;
     title.appendChild(span);
+    const resetBtn = document.createElement('button');
+    resetBtn.className = 'chart-zoom-reset';
+    resetBtn.type = 'button';
+    resetBtn.title = 'reset zoom';
+    resetBtn.textContent = '⤢';
+    resetBtn.hidden = true;
+    title.appendChild(resetBtn);
     panel.appendChild(title);
     const body = document.createElement('div');
     body.className = 'chart-body';
@@ -1732,7 +1739,7 @@
     };
   }
 
-  function makeUplotOpts(name, width, seriesCfg) {
+  function makeUplotOpts(name, width, seriesCfg, resetBtn) {
     const muted = themeColor('muted') || '#8aa0b8';
     const grid = themeColor('grid') || '#1f2630';
     return {
@@ -1757,6 +1764,18 @@
       series: seriesCfg,
       legend: { show: false },
       plugins: [tooltipPlugin()],
+      hooks: {
+        setScale: [
+          (u, key) => {
+            if (!resetBtn || key !== 'x') return;
+            const xData = u.data[0];
+            if (!xData || xData.length < 2) { resetBtn.hidden = true; return; }
+            const sc = u.scales.x;
+            const zoomed = sc.min > xData[0] || sc.max < xData[xData.length - 1];
+            resetBtn.hidden = !zoomed;
+          },
+        ],
+      },
     };
   }
 
@@ -1846,8 +1865,17 @@
 
     if (entry.plot) entry.plot.destroy();
     const body = entry.panel.querySelector('.chart-body');
+    const resetBtn = entry.panel.querySelector('.chart-zoom-reset');
+    if (resetBtn) resetBtn.hidden = true;
     const width = body.clientWidth || 400;
-    entry.plot = new uPlot(makeUplotOpts(name, width, seriesCfg), data, body);
+    entry.plot = new uPlot(makeUplotOpts(name, width, seriesCfg, resetBtn), data, body);
+    if (resetBtn) {
+      resetBtn.onclick = () => {
+        const xData = entry.plot && entry.plot.data[0];
+        if (!xData || !xData.length) return;
+        entry.plot.setScale('x', { min: xData[0], max: xData[xData.length - 1] });
+      };
+    }
     entry.configKey = configKey;
     updateAnnotTable(name, entry, seriesCfg, seriesStats);
   }

From dd3d97ad49af99b8eb5833e67fb41d9066ed70b8 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 09:25:44 -0400
Subject: [PATCH 11/31] added resizeable sidebar

---
 stable_pretraining/web/assets/app.css    | 17 ++++-
 stable_pretraining/web/assets/app.js     | 82 ++++++++++++++++++++++++
 stable_pretraining/web/assets/index.html |  1 +
 3 files changed, 97 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 31e5edc86..e6a957441 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -67,10 +67,19 @@ body {
 
 #app {
   display: grid;
-  grid-template-columns: 300px 1fr;
+  grid-template-columns: 300px 6px 1fr;
   grid-template-rows: 46px 1fr;
   height: 100vh;
 }
+#sidebar-resizer {
+  grid-column: 2; grid-row: 2;
+  cursor: col-resize;
+  background: var(--border);
+  transition: background 0.15s;
+  z-index: 10;
+}
+#sidebar-resizer:hover,
+#sidebar-resizer.dragging { background: var(--accent); }
 
 #topbar {
   grid-column: 1 / -1;
@@ -111,9 +120,10 @@ body {
 /* ---- sidebar -------------------------------------------------------- */
 
 #sidebar {
+  grid-column: 1; grid-row: 2;
   border-right: 1px solid var(--border);
   background: var(--surface);
-  overflow-y: auto;
+  overflow: hidden;
   padding: 10px;
   display: flex; flex-direction: column; gap: 8px;
 }
@@ -230,7 +240,7 @@ body {
 }
 #sidebar-actions button:hover { color: var(--text); border-color: var(--border-strong); }
 
-#run-list { display: flex; flex-direction: column; gap: 1px; }
+#run-list { display: flex; flex-direction: column; gap: 1px; flex: 1; overflow-y: auto; min-height: 0; }
 .run-item {
   display: flex; align-items: center; gap: 8px;
   padding: 5px 7px; border-radius: 4px;
@@ -331,6 +341,7 @@ body {
 /* ---- main panel: tabs + figures/logs ------------------------------- */
 
 #main-panel {
+  grid-column: 3; grid-row: 2;
   display: flex;
   flex-direction: column;
   min-width: 0;          /* let children scroll horizontally if needed */
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index c6d840c3f..014deba5b 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -48,6 +48,12 @@
 
   const SYNC_KEY = 'sptweb-x';
 
+  // Virtual scroll state — set by renderRunList when the flat list is large.
+  const VSCROLL_THRESHOLD = 300;
+  const VSCROLL_OVERSCAN  = 5;
+  let _rowHeight  = 34;
+  let _vScrollState = null;  // { el, runs } | null
+
   // ---- theme -----------------------------------------------------------
 
   const SUN_SVG = '<svg viewBox="0 0 16 16" width="14" height="14" fill="none" stroke="currentColor" stroke-width="1.4" stroke-linecap="round">'
@@ -791,6 +797,15 @@
       `${state.runs.size} run${state.runs.size === 1 ? '' : 's'}` +
       (filtered.length !== state.runs.size ? ` (${filtered.length} shown)` : '');
 
+    // Virtual scroll: flat list only (groups defeat windowing).
+    if (groups.length === 0 && sorted.length > VSCROLL_THRESHOLD) {
+      _vScrollState = { el, runs: sorted };
+      state._lastListKey = null;  // force full rebuild next time we drop below threshold
+      _applyVScrollWindow();
+      return;
+    }
+    _vScrollState = null;
+
     // Skip rebuilding the sidebar DOM when the *structure* (group keys,
     // run order, run count) hasn't changed. Background SSE updates fire
     // every few seconds during training; rebuilding 2 000+ rows on every
@@ -2885,6 +2900,69 @@
     }
   }
 
+  function initSidebarResize() {
+    const app     = document.getElementById('app');
+    const resizer = document.getElementById('sidebar-resizer');
+    const sidebar = document.getElementById('sidebar');
+
+    const saved = parseInt(localStorage.getItem('sptSidebarWidth'), 10);
+    if (saved && saved > 100 && saved < 800) {
+      app.style.gridTemplateColumns = `${saved}px 6px 1fr`;
+    }
+
+    resizer.addEventListener('mousedown', e => {
+      e.preventDefault();
+      const startX = e.clientX;
+      const startW = sidebar.getBoundingClientRect().width;
+      resizer.classList.add('dragging');
+
+      function onMove(e) {
+        const newW = Math.max(160, Math.min(600, startW + (e.clientX - startX)));
+        app.style.gridTemplateColumns = `${newW}px 6px 1fr`;
+      }
+      function onUp() {
+        resizer.classList.remove('dragging');
+        const cols = getComputedStyle(app).gridTemplateColumns;
+        const w = parseInt(cols, 10);
+        if (w) localStorage.setItem('sptSidebarWidth', String(w));
+        document.removeEventListener('mousemove', onMove);
+        document.removeEventListener('mouseup', onUp);
+      }
+      document.addEventListener('mousemove', onMove);
+      document.addEventListener('mouseup', onUp);
+    });
+  }
+
+  function _applyVScrollWindow() {
+    if (!_vScrollState) return;
+    const { el, runs } = _vScrollState;
+    const rowH     = _rowHeight;
+    const scrollTop = el.scrollTop;
+    const clientH  = el.clientHeight || 500;
+
+    const firstIdx = Math.max(0, Math.floor(scrollTop / rowH) - VSCROLL_OVERSCAN);
+    const lastIdx  = Math.min(runs.length - 1,
+      Math.ceil((scrollTop + clientH) / rowH) + VSCROLL_OVERSCAN);
+
+    const frag = document.createDocumentFragment();
+    const topSpacer = document.createElement('div');
+    topSpacer.style.height = `${firstIdx * rowH}px`;
+    frag.appendChild(topSpacer);
+    for (let i = firstIdx; i <= lastIdx; i++) frag.appendChild(makeRunRow(runs[i]));
+    const botSpacer = document.createElement('div');
+    botSpacer.style.height = `${Math.max(0, runs.length - 1 - lastIdx) * rowH}px`;
+    frag.appendChild(botSpacer);
+
+    el.replaceChildren(frag);
+
+    // Update measured row height from the first rendered item.
+    const row0 = el.querySelector('.run-item');
+    if (row0) {
+      const h = row0.offsetHeight;
+      if (h > 1) _rowHeight = h + 1; // +1 for the gap
+    }
+  }
+
   function exportMetricsCSV() {
     const visIds = effectivelyVisible();
     const metricNames = visibleMetricNames();
@@ -3128,6 +3206,10 @@
     wireControls();
     syncControlsToState();
     initTheme();
+    initSidebarResize();
+    document.getElementById('run-list').addEventListener('scroll', () => {
+      if (_vScrollState) requestAnimationFrame(_applyVScrollWindow);
+    }, { passive: true });
     // Tick every minute to keep duration displays and stale indicators current.
     setInterval(() => {
       for (const el of document.querySelectorAll('.run-dur[data-run-id]')) {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 0e5611044..ca5c2874b 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -48,6 +48,7 @@
     </div>
     <div id="run-list"></div>
   </aside>
+  <div id="sidebar-resizer" aria-hidden="true"></div>
   <main id="main-panel">
     <nav id="tabs" class="tabs" role="tablist">
       <button class="tab active" data-tab="figures" type="button">figures</button>

From 9095392d3df57a418dc28dd911504d8d67397dde Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 09:34:56 -0400
Subject: [PATCH 12/31] added keyboard shortcuts

---
 stable_pretraining/web/assets/app.css    | 48 ++++++++++++++++++++
 stable_pretraining/web/assets/app.js     | 57 ++++++++++++++++++++++++
 stable_pretraining/web/assets/index.html | 19 ++++++++
 3 files changed, 124 insertions(+)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index e6a957441..05d7a8c9d 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -981,6 +981,54 @@ body {
 
 /* ---- detail (config) modal ----------------------------------------- */
 
+/* ---- keyboard shortcuts popover ------------------------------------ */
+#kbd-overlay {
+  position: fixed; inset: 0;
+  background: var(--overlay);
+  display: flex; align-items: center; justify-content: center;
+  z-index: 200; backdrop-filter: blur(2px);
+}
+#kbd-overlay[hidden] { display: none; }
+#kbd-card {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  width: min(400px, 92vw);
+  box-shadow: var(--shadow);
+  padding: 0 0 16px;
+}
+.kbd-header {
+  display: flex; align-items: center; justify-content: space-between;
+  padding: 12px 14px;
+  border-bottom: 1px solid var(--border);
+  font-weight: 600; font-size: 13px; color: var(--text-strong);
+}
+#kbd-close {
+  background: none; border: none; color: var(--muted);
+  font-size: 22px; cursor: pointer; line-height: 1;
+  padding: 0 4px; font-family: inherit;
+}
+#kbd-close:hover { color: var(--text); }
+.kbd-table {
+  width: 100%; border-collapse: collapse;
+  margin-top: 8px; font-size: 13px;
+}
+.kbd-table td { padding: 6px 16px; vertical-align: middle; }
+.kbd-table td:first-child { white-space: nowrap; text-align: right; width: 1%; }
+.kbd-table td:last-child { color: var(--muted); padding-left: 20px; }
+kbd {
+  display: inline-block;
+  background: var(--surface-2);
+  border: 1px solid var(--border-strong);
+  border-radius: 4px;
+  padding: 1px 6px;
+  font-size: 11px;
+  font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+  color: var(--text-strong);
+  box-shadow: 0 1px 0 var(--border-strong);
+  line-height: 1.6;
+}
+
 #detail-overlay {
   position: fixed; inset: 0;
   background: var(--overlay);
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 014deba5b..5404d255c 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -3060,9 +3060,66 @@
       if (e.key === 'Escape') {
         const box = document.getElementById('image-lightbox');
         if (box && !box.hidden) closeLightbox();
+        const kbd = document.getElementById('kbd-overlay');
+        if (kbd && !kbd.hidden) kbd.hidden = true;
       }
     });
 
+    // Global keyboard shortcuts — skip when focus is inside an input/textarea.
+    document.addEventListener('keydown', e => {
+      const tag = document.activeElement && document.activeElement.tagName;
+      const inInput = tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT';
+
+      if (e.key === '?' && !inInput) {
+        e.preventDefault();
+        const kbd = document.getElementById('kbd-overlay');
+        if (kbd) kbd.hidden = !kbd.hidden;
+        return;
+      }
+      if (inInput) return;
+
+      const tabs = ['figures', 'out', 'err', 'table'];
+
+      if (e.key === '/') {
+        e.preventDefault();
+        const ms = document.getElementById('metric-search');
+        if (ms) {
+          setActiveTab('figures');
+          ms.focus(); ms.select();
+          ms.addEventListener('keydown', function esc(ke) {
+            if (ke.key === 'Escape') { ke.preventDefault(); ms.blur(); ms.removeEventListener('keydown', esc); }
+          });
+        }
+      } else if (e.key === 'r') {
+        e.preventDefault();
+        const rs = document.getElementById('run-search');
+        if (rs) {
+          rs.focus();
+          rs.addEventListener('keydown', function esc(ke) {
+            if (ke.key === 'Escape') { ke.preventDefault(); rs.blur(); rs.removeEventListener('keydown', esc); }
+          });
+        }
+      } else if (e.key === 't') {
+        e.preventDefault();
+        const idx = tabs.indexOf(state.activeTab);
+        setActiveTab(tabs[(idx + 1) % tabs.length]);
+      } else if (e.key === 'A' && e.shiftKey) {
+        e.preventDefault();
+        setAllVisible(true);
+      } else if (e.key === 'C' && e.shiftKey) {
+        e.preventDefault();
+        setAllVisible(false);
+      }
+    });
+
+    document.getElementById('kbd-close')?.addEventListener('click', () => {
+      document.getElementById('kbd-overlay').hidden = true;
+    });
+    document.getElementById('kbd-overlay')?.addEventListener('click', e => {
+      if (e.target === document.getElementById('kbd-overlay'))
+        document.getElementById('kbd-overlay').hidden = true;
+    });
+
     // Metric search (debounced; force a tree rebuild because the set of
     // visible metrics changes).
     const ms = document.getElementById('metric-search');
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index ca5c2874b..66726788c 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -130,6 +130,25 @@
     <div id="detail-body"></div>
   </div>
 </div>
+<div id="kbd-overlay" hidden role="dialog" aria-modal="true" aria-label="keyboard shortcuts">
+  <div id="kbd-card">
+    <div class="kbd-header">
+      <span>keyboard shortcuts</span>
+      <button id="kbd-close" type="button" title="close (Esc)" aria-label="close">×</button>
+    </div>
+    <table class="kbd-table">
+      <tbody>
+        <tr><td><kbd>/</kbd></td><td>focus metric search</td></tr>
+        <tr><td><kbd>r</kbd></td><td>focus run search</td></tr>
+        <tr><td><kbd>t</kbd></td><td>cycle tabs</td></tr>
+        <tr><td><kbd>Shift</kbd>+<kbd>A</kbd></td><td>select all visible runs</td></tr>
+        <tr><td><kbd>Shift</kbd>+<kbd>C</kbd></td><td>clear all runs</td></tr>
+        <tr><td><kbd>Esc</kbd></td><td>close modal / popover</td></tr>
+        <tr><td><kbd>?</kbd></td><td>show this help</td></tr>
+      </tbody>
+    </table>
+  </div>
+</div>
 <script src="https://cdn.jsdelivr.net/npm/uplot@1.6.31/dist/uPlot.iife.min.js"></script>
 <script src="/assets/app.js"></script>
 </body>

From 4d1b488ced3288eb41f24dd021cf0e731d751f45 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 09:44:05 -0400
Subject: [PATCH 13/31] added functionality to add new tags

---
 stable_pretraining/web/assets/app.css |  26 +++++++
 stable_pretraining/web/assets/app.js  | 104 +++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 4 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 05d7a8c9d..514d4f10a 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -1106,6 +1106,32 @@ kbd {
 .kv-val.null { color: var(--muted-2); font-style: italic; }
 
 /* ---- notes textarea ---------------------------------------------------- */
+.tags-section { margin-bottom: 6px; }
+.tag-pills {
+  display: flex; flex-wrap: wrap; gap: 5px; align-items: center;
+  padding: 2px 0;
+}
+.tag-pill {
+  display: inline-flex; align-items: center; gap: 3px;
+  background: var(--accent-bg); border: 1px solid var(--accent-border);
+  border-radius: 10px; padding: 2px 8px 2px 10px;
+  font-size: 11px; color: var(--text);
+}
+.tag-remove {
+  background: none; border: none; color: var(--muted);
+  cursor: pointer; padding: 0; line-height: 1; font-size: 14px;
+  font-family: inherit;
+}
+.tag-remove:hover { color: var(--text); }
+.tag-add-input {
+  border: 1px dashed var(--border-strong); border-radius: 10px;
+  background: none; color: var(--text);
+  padding: 2px 10px; font-size: 11px; font-family: inherit;
+  width: 70px; transition: width 0.15s, border-color 0.15s;
+}
+.tag-add-input:focus { outline: none; border-color: var(--accent); width: 120px; }
+.tag-add-input::placeholder { color: var(--muted-2); }
+
 .notes-section { margin-bottom: 6px; }
 .notes-textarea {
   width: 100%; box-sizing: border-box;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 5404d255c..9bc51c683 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -2538,11 +2538,15 @@
 
     const body = document.getElementById('detail-body');
 
-    // Reuse the notes section if the user is actively editing to avoid losing cursor.
+    // Reuse the notes/tags sections if the user is actively editing to avoid losing cursor.
     const prevNotes = body.querySelector('.notes-section');
     const editing = prevNotes && document.activeElement === prevNotes.querySelector('textarea');
     const notesSec = editing ? prevNotes : _makeNotesSection(r);
 
+    const prevTags = body.querySelector('.tags-section');
+    const taggingActive = prevTags && document.activeElement === prevTags.querySelector('.tag-add-input');
+    const tagsSec = taggingActive ? prevTags : _makeTagsSection(r);
+
     const now = Date.now() / 1000;
     let durSecs = null;
     if (r.created_at) {
@@ -2555,7 +2559,6 @@
       started: fmtTime(r.created_at),
       ended: r.ended_at ? fmtTime(r.ended_at) : null,
       duration: fmtDuration(durSecs),
-      tags: (r.tags || []).join(', ') || null,
       checkpoint_path: r.checkpoint_path,
     };
 
@@ -2574,9 +2577,9 @@
       empty.style.padding = '16px 0';
       empty.style.textAlign = 'center';
       empty.textContent = filter ? 'no keys match the filter' : 'no config recorded';
-      body.replaceChildren(notesSec, empty);
+      body.replaceChildren(notesSec, tagsSec, empty);
     } else {
-      body.replaceChildren(notesSec, ...sections);
+      body.replaceChildren(notesSec, tagsSec, ...sections);
     }
 
     // Fit textarea height to content. Deferred so the browser has laid out the
@@ -2625,6 +2628,99 @@
     return sec;
   }
 
+  function allExistingTags() {
+    const tags = new Set();
+    for (const r of state.runs.values()) {
+      for (const t of r.tags || []) tags.add(t);
+    }
+    return [...tags].sort();
+  }
+
+  async function _patchTags(runId, newTags) {
+    const run = state.runs.get(runId);
+    if (!run) return;
+    const prev = [...(run.tags || [])];
+    run.tags = newTags;
+    renderDetail();
+    renderRunList();
+    try {
+      await patchRunMeta(runId, { tags: newTags });
+    } catch (e) {
+      console.warn('tags patch failed', e);
+      run.tags = prev;
+      renderDetail();
+      renderRunList();
+    }
+  }
+
+  function _makeTagsSection(r) {
+    const sec = document.createElement('section');
+    sec.className = 'detail-section tags-section';
+    const h = document.createElement('h3');
+    h.textContent = 'tags';
+    sec.appendChild(h);
+
+    const pills = document.createElement('div');
+    pills.className = 'tag-pills';
+
+    for (const tag of r.tags || []) {
+      const pill = document.createElement('span');
+      pill.className = 'tag-pill';
+      const label = document.createElement('span');
+      label.textContent = tag;
+      const rm = document.createElement('button');
+      rm.type = 'button';
+      rm.className = 'tag-remove';
+      rm.title = `remove "${tag}"`;
+      rm.textContent = '×';
+      rm.addEventListener('click', () => {
+        _patchTags(r.run_id, (r.tags || []).filter(t => t !== tag));
+      });
+      pill.append(label, rm);
+      pills.appendChild(pill);
+    }
+
+    const listId = 'tag-suggestions-datalist';
+    let datalist = document.getElementById(listId);
+    if (!datalist) {
+      datalist = document.createElement('datalist');
+      datalist.id = listId;
+      document.body.appendChild(datalist);
+    }
+    datalist.replaceChildren(
+      ...allExistingTags()
+        .filter(t => !(r.tags || []).includes(t))
+        .map(t => { const o = document.createElement('option'); o.value = t; return o; })
+    );
+
+    const input = document.createElement('input');
+    input.type = 'text';
+    input.className = 'tag-add-input';
+    input.placeholder = '+ tag';
+    input.setAttribute('list', listId);
+    input.autocomplete = 'off';
+    input.spellcheck = false;
+
+    function commitInput() {
+      const val = input.value.trim();
+      input.value = '';
+      if (!val) return;
+      const current = r.tags || [];
+      if (current.includes(val)) return;
+      _patchTags(r.run_id, [...current, val]);
+    }
+
+    input.addEventListener('keydown', e => {
+      if (e.key === 'Enter' || e.key === ',') { e.preventDefault(); commitInput(); }
+      else if (e.key === 'Escape') { input.value = ''; input.blur(); }
+    });
+    input.addEventListener('blur', commitInput);
+
+    pills.appendChild(input);
+    sec.appendChild(pills);
+    return sec;
+  }
+
   // ---- SSE --------------------------------------------------------------
 
   function startSSE() {

From c95b0cf34014f0cbb8dfe43f42c7d6640465311c Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 10:24:35 -0400
Subject: [PATCH 14/31] fixing format

---
 RELEASES.rst                   | 81 ++++++++++++++++++++++++++++++++++
 stable_pretraining/web/scan.py |  4 +-
 2 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/RELEASES.rst b/RELEASES.rst
index 8794bd94d..fdbd02257 100644
--- a/RELEASES.rst
+++ b/RELEASES.rst
@@ -57,6 +57,87 @@ Unreleased
 - New API pages: :doc:`api/registry` (RegistryLogger + Registry query)
   and :doc:`api/web` (``serve`` entry point).
 
+**spt web — viewer improvements**
+
+- **State persistence & shareable URLs**: all interactive state
+  (selected runs, filters, group-by, sort, x-axis, smoothing, log-y, active
+  tab, theme) is serialised to ``localStorage`` on every mutation and restored
+  on load. The URL fragment is kept in sync (``#runs=…&tab=…``) so copying the
+  address bar produces a link that reopens the exact same selection.
+
+- **Chart value annotations**: each metric chart now has a compact table below
+  it showing the last and best value for every visible series. The best row is
+  highlighted; clicking a row toggles that series on/off. Lower-is-better is
+  inferred automatically from metric names containing ``loss``, ``err``,
+  ``perplexity``, or ``ppl``.
+
+- **Runs table view**: a fourth **table** tab renders a horizontally-scrollable
+  grid with one row per visible run and one column per hparam/summary key. Cells
+  whose values differ across runs are highlighted in amber; identical cells are
+  dimmed. Columns are sortable by click and filterable by a search box. The
+  run-ID column and header row are sticky.
+
+- **Run display names**: the sidebar and table now show ``display_name`` as the
+  primary label with the raw path shown as a dimmed hint. Double-clicking the
+  label opens an inline editor; changes are persisted via ``PATCH /api/run-meta``.
+
+- **Notes editing**: the run detail modal has an editable notes textarea that
+  auto-resizes to its content. Notes are saved on blur or ``Ctrl+Enter`` via
+  ``PATCH /api/run-meta``.
+
+- **Log auto-refresh / live tail**: the ``.out`` and ``.err`` tabs auto-refresh
+  every 10 s while the selected run is running or stale, showing a pulsing
+  **live** badge. A pause button stops the timer; a refresh button triggers a
+  manual fetch. Scroll position is preserved when the user has scrolled up.
+
+- **Elapsed time / duration**: each run row and the detail modal now show the
+  elapsed or total duration. Running runs tick forward every 60 s without a full
+  re-render. ``RegistryLogger.finalize()`` now records ``ended_at`` in the
+  sidecar so completed durations survive restarts.
+
+- **Heartbeat staleness**: runs whose heartbeat file is more than 5 minutes old
+  are flagged as **stale** with an amber ⚠ indicator in the sidebar, topbar
+  stats, detail modal, figures overview, and activity timeline. Filters,
+  group-by, and sort all treat stale as a distinct status value via a centralised
+  ``effectiveStatus()`` helper.
+
+- **Scatter plot**: the figures tab renders a scatter plot below the metric
+  charts when two or more runs are visible. X and Y axes are independently
+  selectable from any numeric ``hparams.*`` or ``summary.*`` key across visible
+  runs. Each run contributes one point (its final summary scalar). Axis
+  selection is persisted to ``localStorage``.
+
+- **CSV export**: a **download CSV** button in the figures toolbar downloads all
+  visible metrics for the currently selected runs as a flat CSV
+  (``run_id, run_name, metric, step, epoch, value``). Only the runs and metrics
+  currently in view (respecting the metric-search filter) are included. No
+  server round-trip — the data is built entirely from the in-memory metrics
+  cache.
+
+- **Zoom reset**: drag-selecting a region on any chart zooms all charts
+  simultaneously (shared sync key). A **⤢** reset button appears in the chart
+  title bar after zooming and resets all charts to their full x-range in one
+  click. The drag-selection region is now styled with an accent-coloured border
+  and fill.
+
+- **Sidebar resize + virtual scroll**: the sidebar can be dragged to any width
+  between 160 px and 600 px; the chosen width is persisted. The
+  search/filter/sort controls are now fixed at the top of the sidebar while the
+  run list scrolls independently below them. When more than 300 ungrouped runs
+  are visible, the run list switches to a virtual-scroll window so SSE updates
+  remain smooth at scale.
+
+- **Keyboard shortcuts**: ``/`` focuses metric search, ``r`` focuses run
+  search, ``t`` cycles tabs, ``Shift+A`` selects all, ``Shift+C`` clears all,
+  ``?`` opens a help popover listing all shortcuts. ``Esc`` exits focused inputs
+  and dismisses all modals and popovers.
+
+- **Tag editing**: run tags are now displayed as editable pills in the detail
+  modal. Clicking **star** on a pill removes the tag; a dashed ``+ tag`` input
+  with autocomplete (populated from tags on other runs) adds new ones. All
+  changes are persisted via ``PATCH /api/run-meta`` with optimistic updates and
+  revert on failure.
+
 Version 0.1
 -----------
 
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index 088eae289..934987e91 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -393,9 +393,7 @@ def _serialize(run: _Run) -> dict:
         # display_name: explicit sidecar field wins; fall back to the last
         # path component of run_id so short IDs stay readable by default.
         display_name = (
-            s.get("display_name")
-            or run.run_id.rsplit("/", 1)[-1]
-            or run.run_id
+            s.get("display_name") or run.run_id.rsplit("/", 1)[-1] or run.run_id
         )
         try:
             hb_at = heartbeat_mtime(run.run_dir)

From bd06035086894074270a48f9ea21f16b83fc5018 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 15:22:50 -0400
Subject: [PATCH 15/31] made it so that removing a zoom, zooms out of all
 graphs

---
 stable_pretraining/web/assets/app.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 9bc51c683..5c5dbbcec 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -1886,9 +1886,12 @@
     entry.plot = new uPlot(makeUplotOpts(name, width, seriesCfg, resetBtn), data, body);
     if (resetBtn) {
       resetBtn.onclick = () => {
-        const xData = entry.plot && entry.plot.data[0];
-        if (!xData || !xData.length) return;
-        entry.plot.setScale('x', { min: xData[0], max: xData[xData.length - 1] });
+        for (const [, e] of state.charts) {
+          if (!e.plot) continue;
+          const xData = e.plot.data[0];
+          if (!xData || !xData.length) continue;
+          e.plot.setScale('x', { min: xData[0], max: xData[xData.length - 1] });
+        }
       };
     }
     entry.configKey = configKey;

From 2599ffeb65a37dae74c43a843198f59f7dc24824 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 15:32:09 -0400
Subject: [PATCH 16/31] added button to table that will hide all the things
 that are the same

---
 stable_pretraining/web/assets/app.css    |  1 +
 stable_pretraining/web/assets/app.js     | 45 ++++++++++++++++++++----
 stable_pretraining/web/assets/index.html |  1 +
 3 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 514d4f10a..cdb40871b 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -115,6 +115,7 @@ body {
   display: inline-flex; align-items: center; justify-content: center;
 }
 .icon-btn:hover { color: var(--accent); border-color: var(--accent); }
+.icon-btn.active { color: var(--accent); border-color: var(--accent); background: var(--accent-bg); }
 .icon-btn svg { display: block; }
 
 /* ---- sidebar -------------------------------------------------------- */
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 5c5dbbcec..0b62622a6 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -38,6 +38,7 @@
     logLivePaused: { out: false, err: false },
     scatterX: null,   // 'hparams.lr' | 'summary.val_acc' | …
     scatterY: null,
+    tableHideSame: false,
   };
 
   // setInterval handles for log auto-refresh; null when not running.
@@ -112,6 +113,7 @@
       tableColSearch: state.tableColSearch,
       scatterX: state.scatterX,
       scatterY: state.scatterY,
+      tableHideSame: state.tableHideSame,
       activeTab: state.activeTab,
       theme: state.theme,
     };
@@ -157,6 +159,7 @@
       if (typeof saved.tableColSearch === 'string') state.tableColSearch = saved.tableColSearch;
       if (typeof saved.scatterX === 'string') state.scatterX = saved.scatterX;
       if (typeof saved.scatterY === 'string') state.scatterY = saved.scatterY;
+      if (typeof saved.tableHideSame === 'boolean') state.tableHideSame = saved.tableHideSame;
       if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
       if (typeof saved.theme === 'string') state.theme = saved.theme;
       if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
@@ -183,6 +186,11 @@
     if (msEl) msEl.value = state.metricSearch;
     const tcsEl = document.getElementById('table-col-search');
     if (tcsEl) tcsEl.value = state.tableColSearch;
+    const hsBtn = document.getElementById('table-hide-same');
+    if (hsBtn) {
+      hsBtn.textContent = state.tableHideSame ? 'show same' : 'hide same';
+      hsBtn.classList.toggle('active', state.tableHideSame);
+    }
     // Apply active tab to DOM directly — calling setActiveTab() would fire
     // saveState() while state.visible is still empty (runs not yet loaded).
     for (const btn of document.querySelectorAll('#tabs .tab')) {
@@ -1179,13 +1187,7 @@
 
     // Filter columns by search.
     const q = (state.tableColSearch || '').trim().toLowerCase();
-    const cols = q ? allCols.filter(c => c.label.toLowerCase().includes(q)) : allCols;
-
-    if (cnt) {
-      cnt.textContent = cols.length === allCols.length
-        ? `${allCols.length} cols`
-        : `${cols.length} / ${allCols.length} cols`;
-    }
+    let cols = q ? allCols.filter(c => c.label.toLowerCase().includes(q)) : allCols;
 
     // Determine which columns have any differing values.
     const diffSet = new Set();
@@ -1197,6 +1199,23 @@
       if (new Set(vals).size > 1) diffSet.add(col.id);
     }
 
+    // Optionally hide columns where all visible runs have the same value.
+    if (state.tableHideSame) {
+      cols = cols.filter(c => diffSet.has(c.id));
+    }
+
+    if (cnt) {
+      const shown = cols.length;
+      const total = allCols.length;
+      if (shown === total) {
+        cnt.textContent = `${total} cols`;
+      } else if (state.tableHideSame && !q) {
+        cnt.textContent = `${shown} diff / ${total} cols`;
+      } else {
+        cnt.textContent = `${shown} / ${total} cols`;
+      }
+    }
+
     // Apply table sort.
     let sortedRuns = [...runs];
     const ts = state.tableSort;
@@ -3241,6 +3260,18 @@
       }, 80));
     }
 
+    // Table: hide same-value columns toggle
+    const hideSameBtn = document.getElementById('table-hide-same');
+    if (hideSameBtn) {
+      hideSameBtn.addEventListener('click', () => {
+        state.tableHideSame = !state.tableHideSame;
+        hideSameBtn.textContent = state.tableHideSame ? 'show same' : 'hide same';
+        hideSameBtn.classList.toggle('active', state.tableHideSame);
+        saveState();
+        renderRunsTable();
+      });
+    }
+
     // Log tab selectors / refresh buttons
     const outSel = document.getElementById('logs-stream-selector');
     if (outSel) outSel.addEventListener('change', () => {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 66726788c..1514f6724 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -102,6 +102,7 @@
       <div id="table-controls">
         <input id="table-col-search" type="search" placeholder="filter columns..." autocomplete="off" spellcheck="false">
         <span id="table-col-count" class="table-col-count"></span>
+        <button id="table-hide-same" class="icon-btn" type="button" title="hide columns with identical values across all visible runs">hide same</button>
       </div>
       <div class="runs-table-wrap"></div>
     </div>

From e9dffb4bcf5ec9793a718ffb4d3faef0322be8ae Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Fri, 5 Jun 2026 16:52:56 -0400
Subject: [PATCH 17/31] functionality to download tables

---
 stable_pretraining/web/assets/app.css | 15 ++++++++---
 stable_pretraining/web/assets/app.js  | 38 +++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index cdb40871b..d90e6d7ec 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -437,8 +437,8 @@ body {
 }
 .log-live-badge {
   font-size: 10px; font-weight: 600; letter-spacing: 0.4px;
-  color: var(--success, #48bb78);
-  border: 1px solid var(--success, #48bb78);
+  color: var(--bad);
+  border: 1px solid var(--bad);
   border-radius: 3px;
   padding: 1px 5px;
   text-transform: uppercase;
@@ -642,9 +642,18 @@ body {
   overflow: hidden; white-space: nowrap; text-overflow: ellipsis;
 }
 .chart-body { width: 100%; }
-.chart-zoom-reset {
+.chart-download-btn {
   flex-shrink: 0;
   margin-left: auto;
+  opacity: 0;
+  transition: opacity 0.15s;
+  font-size: 11px;
+  padding: 1px 5px;
+}
+.chart-panel:hover .chart-download-btn { opacity: 1; }
+
+.chart-zoom-reset {
+  flex-shrink: 0;
   background: none;
   border: 1px solid var(--border);
   border-radius: 4px;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 0b62622a6..19be72190 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -1319,6 +1319,37 @@
     wrap.replaceChildren(table);
   }
 
+  function downloadChartPNG(metricName, displayName) {
+    const entry = state.charts.get(metricName);
+    if (!entry || !entry.plot) return;
+    const srcCanvas = entry.plot.ctx.canvas;
+    const dpr = window.devicePixelRatio || 1;
+    const PAD = Math.round(12 * dpr);
+    const TITLE_H = Math.round(26 * dpr);
+    const off = document.createElement('canvas');
+    off.width  = srcCanvas.width + PAD * 2;
+    off.height = srcCanvas.height + TITLE_H + PAD * 2;
+    const ctx = off.getContext('2d');
+    ctx.fillStyle = themeColor('surface') || '#11151c';
+    ctx.fillRect(0, 0, off.width, off.height);
+    ctx.font = `bold ${Math.round(13 * dpr)}px ui-monospace, SFMono-Regular, Menlo, monospace`;
+    ctx.fillStyle = themeColor('text-strong') || '#f1f5f9';
+    ctx.fillText(displayName || metricName, PAD, PAD + Math.round(15 * dpr));
+    ctx.drawImage(srcCanvas, PAD, TITLE_H + PAD);
+    const fileName = (displayName || metricName).replace(/[/\\:*?"<>|]/g, '_') + '.png';
+    off.toBlob(blob => {
+      if (!blob) return;
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = fileName;
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+      URL.revokeObjectURL(url);
+    }, 'image/png');
+  }
+
   function makePanel(fullName, displayName) {
     const panel = document.createElement('div');
     panel.className = 'chart-panel';
@@ -1329,6 +1360,13 @@
     span.textContent = displayName || fullName;
     span.title = fullName;
     title.appendChild(span);
+    const dlBtn = document.createElement('button');
+    dlBtn.className = 'chart-download-btn icon-btn';
+    dlBtn.type = 'button';
+    dlBtn.title = 'download chart as PNG';
+    dlBtn.textContent = '⬇';
+    dlBtn.addEventListener('click', () => downloadChartPNG(fullName, displayName || fullName));
+    title.appendChild(dlBtn);
     const resetBtn = document.createElement('button');
     resetBtn.className = 'chart-zoom-reset';
     resetBtn.type = 'button';

From fef791f190d59bf44dfaf10a70589ab53bc2141f Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sat, 6 Jun 2026 12:12:14 -0400
Subject: [PATCH 18/31] functionality to combine different metrics into a
 single graph for a run

---
 stable_pretraining/web/assets/app.css    |  30 +++
 stable_pretraining/web/assets/app.js     | 223 ++++++++++++++++++++++-
 stable_pretraining/web/assets/index.html |   1 +
 3 files changed, 251 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index d90e6d7ec..b74dc2311 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -652,6 +652,36 @@ body {
 }
 .chart-panel:hover .chart-download-btn { opacity: 1; }
 
+.chart-combine-toggle {
+  flex-shrink: 0;
+  opacity: 0;
+  transition: opacity 0.15s;
+  font-size: 13px;
+  font-weight: 700;
+  padding: 1px 6px;
+}
+.chart-panel:hover .chart-combine-toggle { opacity: 0.6; }
+.chart-combine-toggle.active { opacity: 1; }
+
+.chart-remove-btn {
+  flex-shrink: 0;
+  font-size: 14px;
+  padding: 1px 7px;
+}
+
+.combined-chart-panel {
+  border-color: var(--accent);
+  border-width: 2px;
+}
+
+.combined-charts-section {
+  display: flex;
+  flex-direction: column;
+  gap: 16px;
+  margin-bottom: 16px;
+}
+.combined-charts-section:empty { display: none; }
+
 .chart-zoom-reset {
   flex-shrink: 0;
   background: none;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 19be72190..8a3c896fb 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -39,6 +39,8 @@
     scatterX: null,   // 'hparams.lr' | 'summary.val_acc' | …
     scatterY: null,
     tableHideSame: false,
+    combineSelection: new Set(),  // metric names currently selected for combining
+    combinedCharts: [],           // [{id, metrics: string[]}]
   };
 
   // setInterval handles for log auto-refresh; null when not running.
@@ -116,6 +118,7 @@
       tableHideSame: state.tableHideSame,
       activeTab: state.activeTab,
       theme: state.theme,
+      combinedCharts: state.combinedCharts,
     };
     try { localStorage.setItem('spt-web-state', JSON.stringify(snap)); } catch {}
     const ids = [...state.visible].map(id => encodeURIComponent(id)).join(',');
@@ -162,6 +165,7 @@
       if (typeof saved.tableHideSame === 'boolean') state.tableHideSame = saved.tableHideSame;
       if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
       if (typeof saved.theme === 'string') state.theme = saved.theme;
+      if (Array.isArray(saved.combinedCharts)) state.combinedCharts = saved.combinedCharts;
       if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
     }
 
@@ -1077,7 +1081,7 @@
     const mediaTags = visibleMediaTags();
     const allTags = [...new Set([...metrics, ...mediaTags.keys()])];
 
-    if (allTags.length === 0) {
+    if (allTags.length === 0 && state.combinedCharts.length === 0) {
       for (const { plot } of state.charts.values()) plot?.destroy();
       state.charts.clear();
       for (const { panel } of state.mediaPanels.values()) panel.remove();
@@ -1088,15 +1092,19 @@
       return;
     }
 
-    // Tear down chart panels for metrics that vanished.
+    // Tear down chart panels for metrics that vanished (skip combined charts).
     const metricSet = new Set(metrics);
+    let selectionChanged = false;
     for (const [name, entry] of [...state.charts.entries()]) {
+      if (name.startsWith('__combined__/')) continue;
       if (!metricSet.has(name) || mediaTags.has(name)) {
         entry.plot?.destroy();
         entry.panel.remove();
         state.charts.delete(name);
+        if (state.combineSelection.delete(name)) selectionChanged = true;
       }
     }
+    if (selectionChanged) updateCombineSelectionUI();
     // Tear down media panels for tags that vanished.
     for (const [tag, entry] of [...state.mediaPanels.entries()]) {
       if (!mediaTags.has(tag)) {
@@ -1105,6 +1113,14 @@
       }
     }
 
+    // Combined charts section — pinned above the metric tree.
+    let combinedSection = root.querySelector(':scope > .combined-charts-section');
+    if (!combinedSection) {
+      combinedSection = document.createElement('div');
+      combinedSection.className = 'combined-charts-section';
+    }
+    renderCombinedCharts(combinedSection);
+
     // Skip the full tree rebuild when nothing structurally changed.
     // Streaming-metrics chunks fire scheduleRerender many times per
     // second; re-parenting a playing ``<video>`` (or just shifting page
@@ -1120,7 +1136,7 @@
       const container = document.createElement('div');
       container.className = 'metric-tree';
       attachMetricTree(container, tree, '', mediaTags);
-      root.replaceChildren(container);
+      root.replaceChildren(combinedSection, container);
       state._lastTreeKey = treeKey;
 
       // After re-parenting, sizes may have changed. Resize each plot.
@@ -1131,6 +1147,8 @@
           if (w) entry.plot.setSize({ width: w, height: 240 });
         }
       }
+    } else if (root.firstChild !== combinedSection) {
+      root.prepend(combinedSection);
     }
 
     for (const name of metrics) updateChart(name);
@@ -1360,6 +1378,21 @@
     span.textContent = displayName || fullName;
     span.title = fullName;
     title.appendChild(span);
+    const toggleBtn = document.createElement('button');
+    toggleBtn.className = 'chart-combine-toggle icon-btn';
+    toggleBtn.type = 'button';
+    toggleBtn.title = 'select to combine into one chart';
+    toggleBtn.textContent = '+';
+    toggleBtn.addEventListener('click', (e) => {
+      e.stopPropagation();
+      if (state.combineSelection.has(fullName)) {
+        state.combineSelection.delete(fullName);
+      } else {
+        state.combineSelection.add(fullName);
+      }
+      updateCombineSelectionUI();
+    });
+    title.appendChild(toggleBtn);
     const dlBtn = document.createElement('button');
     dlBtn.className = 'chart-download-btn icon-btn';
     dlBtn.type = 'button';
@@ -1690,6 +1723,179 @@
     return n.includes('loss') || n.includes('err') || n.includes('perplexity') || n.includes('ppl');
   }
 
+  // ---- combine-metrics feature ------------------------------------------
+
+  function updateCombineSelectionUI() {
+    const btn = document.getElementById('combine-btn');
+    if (!btn) return;
+    const n = state.combineSelection.size;
+    btn.textContent = `Combine (${n})`;
+    btn.hidden = n < 2;
+    for (const [name, entry] of state.charts.entries()) {
+      if (name.startsWith('__combined__/')) continue;
+      const toggleBtn = entry.panel.querySelector('.chart-combine-toggle');
+      if (toggleBtn) toggleBtn.classList.toggle('active', state.combineSelection.has(name));
+    }
+  }
+
+  function buildCombinedChartData(metricNames, runIds) {
+    const series = [];
+    const xUnion = new Set();
+    const multiRun = runIds.length > 1;
+    for (const metricName of metricNames) {
+      for (const id of runIds) {
+        const m = state.metrics.get(id);
+        if (!m || !m.metrics[metricName]) continue;
+        const col = m.metrics[metricName];
+        const xs = col[state.xAxis];
+        const fallback = (state.xAxis === 'epoch' && !xs?.some(v => v != null)) ? col.step : xs;
+        const xArr = fallback || col.step;
+        const filtered = { id, metricName, xs: [], ys: [] };
+        for (let i = 0; i < col.y.length; i++) {
+          const x = xArr ? xArr[i] : i;
+          if (x == null || !isFinite(x)) continue;
+          filtered.xs.push(x);
+          filtered.ys.push(col.y[i]);
+          xUnion.add(x);
+        }
+        if (filtered.ys.length) series.push(filtered);
+      }
+    }
+    const xs = [...xUnion].sort((a, b) => a - b);
+    const xIdx = new Map();
+    for (let i = 0; i < xs.length; i++) xIdx.set(xs[i], i);
+    const seriesCfg = [{}];
+    const data = [xs];
+    for (const s of series) {
+      const arr = new Array(xs.length).fill(null);
+      for (let i = 0; i < s.xs.length; i++) arr[xIdx.get(s.xs[i])] = s.ys[i];
+      data.push(emaSmooth(arr, state.smoothing));
+      const label = multiRun ? `${s.metricName} · ${s.id}` : s.metricName;
+      seriesCfg.push({
+        label,
+        runId: label,
+        stroke: runColor(multiRun ? `${s.metricName}:${s.id}` : s.metricName),
+        width: 1.6,
+        points: { show: false },
+        spanGaps: true,
+      });
+    }
+    const seriesStats = series.map(s => {
+      const raw = s.ys.filter(v => v != null && isFinite(v));
+      const lastVal = raw.length ? raw[raw.length - 1] : null;
+      const bestVal = raw.length ? Math.max(...raw) : null;
+      const label = multiRun ? `${s.metricName} · ${s.id}` : s.metricName;
+      return { id: label, lastVal, bestVal };
+    });
+    return { data, seriesCfg, seriesStats };
+  }
+
+  function makeCombinedPanel(combinedId, metricNames) {
+    const panel = document.createElement('div');
+    panel.className = 'chart-panel combined-chart-panel';
+    const title = document.createElement('div');
+    title.className = 'chart-title';
+    const span = document.createElement('span');
+    span.className = 'name';
+    const shortNames = metricNames.map(n => n.split('/').pop());
+    span.textContent = 'Combined: ' + shortNames.join(', ');
+    span.title = metricNames.join(', ');
+    title.appendChild(span);
+    const removeBtn = document.createElement('button');
+    removeBtn.className = 'chart-remove-btn icon-btn';
+    removeBtn.type = 'button';
+    removeBtn.title = 'remove combined chart';
+    removeBtn.textContent = '×';
+    removeBtn.addEventListener('click', () => {
+      state.combinedCharts = state.combinedCharts.filter(c => c.id !== combinedId);
+      const key = '__combined__/' + combinedId;
+      const entry = state.charts.get(key);
+      if (entry) { entry.plot?.destroy(); state.charts.delete(key); }
+      panel.remove();
+      saveState();
+      updateCombineSelectionUI();
+    });
+    title.appendChild(removeBtn);
+    const dlBtn = document.createElement('button');
+    dlBtn.className = 'chart-download-btn icon-btn';
+    dlBtn.type = 'button';
+    dlBtn.title = 'download chart as PNG';
+    dlBtn.textContent = '⬇';
+    dlBtn.addEventListener('click', () => {
+      downloadChartPNG('__combined__/' + combinedId, 'Combined: ' + shortNames.join(', '));
+    });
+    title.appendChild(dlBtn);
+    const resetBtn = document.createElement('button');
+    resetBtn.className = 'chart-zoom-reset';
+    resetBtn.type = 'button';
+    resetBtn.title = 'reset zoom';
+    resetBtn.textContent = '⤢';
+    resetBtn.hidden = true;
+    title.appendChild(resetBtn);
+    panel.appendChild(title);
+    const body = document.createElement('div');
+    body.className = 'chart-body';
+    panel.appendChild(body);
+    const annot = document.createElement('div');
+    annot.className = 'chart-annot';
+    panel.appendChild(annot);
+    return panel;
+  }
+
+  function renderCombinedCharts(container) {
+    const visibleIds = effectivelyVisible().sort();
+    const validIds = new Set(state.combinedCharts.map(c => c.id));
+    for (const [key, entry] of [...state.charts.entries()]) {
+      if (!key.startsWith('__combined__/')) continue;
+      const id = key.slice('__combined__/'.length);
+      if (!validIds.has(id)) {
+        entry.plot?.destroy();
+        entry.panel.remove();
+        state.charts.delete(key);
+      }
+    }
+    for (const combined of state.combinedCharts) {
+      const key = '__combined__/' + combined.id;
+      let entry = state.charts.get(key);
+      if (!entry) {
+        const panel = makeCombinedPanel(combined.id, combined.metrics);
+        entry = { panel, plot: null, configKey: '' };
+        state.charts.set(key, entry);
+      }
+      container.appendChild(entry.panel);
+      const { data, seriesCfg, seriesStats } = buildCombinedChartData(combined.metrics, visibleIds);
+      const configKey = JSON.stringify([visibleIds, combined.metrics, state.xAxis, state.logY, state.theme]);
+      if (seriesCfg.length <= 1) {
+        if (entry.plot) { entry.plot.destroy(); entry.plot = null; entry.configKey = ''; }
+        updateAnnotTable(key, entry, [], []);
+        continue;
+      }
+      if (entry.plot && entry.configKey === configKey) {
+        entry.plot.setData(data);
+        updateAnnotTable(key, entry, seriesCfg, seriesStats);
+        continue;
+      }
+      if (entry.plot) entry.plot.destroy();
+      const body = entry.panel.querySelector('.chart-body');
+      const resetBtn = entry.panel.querySelector('.chart-zoom-reset');
+      if (resetBtn) resetBtn.hidden = true;
+      const width = body.clientWidth || 400;
+      entry.plot = new uPlot(makeUplotOpts(key, width, seriesCfg, resetBtn), data, body);
+      if (resetBtn) {
+        resetBtn.onclick = () => {
+          for (const [, e] of state.charts) {
+            if (!e.plot) continue;
+            const xData = e.plot.data[0];
+            if (!xData || !xData.length) continue;
+            e.plot.setScale('x', { min: xData[0], max: xData[xData.length - 1] });
+          }
+        };
+      }
+      entry.configKey = configKey;
+      updateAnnotTable(key, entry, seriesCfg, seriesStats);
+    }
+  }
+
   function fmtTooltipNum(v) {
     if (v == null || !isFinite(v)) return '—';
     if (Number.isInteger(v)) return String(v);
@@ -3187,6 +3393,17 @@
 
     document.getElementById('export-csv').addEventListener('click', exportMetricsCSV);
 
+    document.getElementById('combine-btn').addEventListener('click', () => {
+      if (state.combineSelection.size < 2) return;
+      const id = String(Date.now());
+      const metrics = [...state.combineSelection];
+      state.combinedCharts.push({ id, metrics });
+      state.combineSelection.clear();
+      saveState();
+      scheduleRerender();
+      updateCombineSelectionUI();
+    });
+
     document.getElementById('add-filter-btn').addEventListener('click', () => openFilterDraft(null));
 
     document.getElementById('group-by').addEventListener('change', e => {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 1514f6724..739046d87 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -73,6 +73,7 @@
                autocomplete="off" spellcheck="false">
         <span id="metric-search-status" class="metric-search-status"></span>
         <button id="export-csv" class="icon-btn" type="button" title="export visible metrics as CSV">⬇ CSV</button>
+        <button id="combine-btn" class="icon-btn" type="button" title="combine selected metrics into one chart" hidden>Combine (0)</button>
       </div>
       <div id="charts"><div class="empty-state">select a run on the left to plot its metrics</div></div>
     </div>

From 5eff3a9611406b8e2ec642197909f2b96bed28f7 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sat, 6 Jun 2026 12:27:57 -0400
Subject: [PATCH 19/31] update to light mode, making it more pleasing to the
 eyes

---
 stable_pretraining/web/assets/app.css | 47 ++++++++++++++-------------
 stable_pretraining/web/assets/app.js  | 10 +++---
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index b74dc2311..50507f82e 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -29,31 +29,31 @@
 }
 
 [data-theme="light"] {
-  --bg: #f6f7f9;
-  --surface: #ffffff;
-  --surface-2: #ffffff;
-  --border: #e3e7ed;
-  --border-2: #eef1f5;
-  --border-strong: #cbd2dc;
-  --grid: #eef1f5;
-  --text: #1f2937;
-  --text-strong: #0f172a;
-  --muted: #4b5563;
-  --muted-2: #94a3b8;
-  --muted-3: #cbd5e1;
-  --accent: #0891b2;
-  --accent-bg: rgba(8, 145, 178, 0.08);
-  --accent-bg-strong: rgba(8, 145, 178, 0.18);
-  --accent-border: #67e8f9;
-  --good: #047857;
+  --bg: #f0ede9;
+  --surface: #f9f8f6;
+  --surface-2: #f3f1ee;
+  --border: #e0dbd5;
+  --border-2: #e8e4df;
+  --border-strong: #c9c2b9;
+  --grid: #e4dfd9;
+  --text: #2e2a26;
+  --text-strong: #1a1714;
+  --muted: #524e4a;
+  --muted-2: #78716c;
+  --muted-3: #9a938c;
+  --accent: #4f46e5;
+  --accent-bg: rgba(79, 70, 229, 0.07);
+  --accent-bg-strong: rgba(79, 70, 229, 0.15);
+  --accent-border: rgba(79, 70, 229, 0.30);
+  --good: #16a34a;
   --bad: #dc2626;
-  --warn: #b45309;
-  --info: #6d28d9;
+  --warn: #d97706;
+  --info: #7c3aed;
   --hover: rgba(0, 0, 0, 0.04);
-  --hover-2: rgba(0, 0, 0, 0.06);
+  --hover-2: rgba(0, 0, 0, 0.07);
   --dot-border: rgba(0, 0, 0, 0.10);
-  --overlay: rgba(15, 23, 42, 0.45);
-  --shadow: 0 12px 40px rgba(15, 23, 42, 0.18);
+  --overlay: rgba(28, 25, 23, 0.42);
+  --shadow: 0 12px 40px rgba(0, 0, 0, 0.10);
 }
 
 /* ---- base ----------------------------------------------------------- */
@@ -90,6 +90,7 @@ body {
 }
 .brand { font-weight: 600; letter-spacing: 0.3px; color: var(--text-strong); }
 #status-row { color: var(--muted); font-size: 12px; }
+#run-count { color: var(--text-strong) !important; font-weight: 600; }
 .spacer { flex: 1; }
 .control {
   display: flex; align-items: center; gap: 8px;
@@ -863,7 +864,7 @@ body {
   background: rgba(251, 191, 36, 0.07);
 }
 [data-theme="light"] .runs-table tbody td.cell-diff {
-  background: rgba(180, 83, 9, 0.06);
+  background: rgba(217, 119, 6, 0.08);
 }
 .runs-table tbody td.cell-same { color: var(--muted-3); }
 .rt-run-id {
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 8a3c896fb..f08d7a16f 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -2606,11 +2606,11 @@
 
     const cards = document.createElement('div');
     cards.className = 'stat-cards';
-    cards.appendChild(statCard('total runs', allRuns.length, '#f1f5f9'));
-    cards.appendChild(statCard('running',   counts.running   || 0, '#22d3ee'));
-    if (counts.stale) cards.appendChild(statCard('stale', counts.stale, '#f59e0b'));
-    cards.appendChild(statCard('completed', counts.completed || 0, '#34d399'));
-    cards.appendChild(statCard('failed',    counts.failed    || 0, '#f87171'));
+    cards.appendChild(statCard('total runs', allRuns.length, themeColor('text-strong')));
+    cards.appendChild(statCard('running',   counts.running   || 0, themeColor('accent')));
+    if (counts.stale) cards.appendChild(statCard('stale', counts.stale, themeColor('warn')));
+    cards.appendChild(statCard('completed', counts.completed || 0, themeColor('good')));
+    cards.appendChild(statCard('failed',    counts.failed    || 0, themeColor('bad')));
     wrap.appendChild(cards);
 
     const activity = document.createElement('div');

From 58faa18171f123267941c7b6e0d74dd36cea237d Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 13:28:48 -0400
Subject: [PATCH 20/31] feature to be able to choose if min or max is better
 for a graph

---
 stable_pretraining/web/assets/app.css | 11 +++++++++
 stable_pretraining/web/assets/app.js  | 33 +++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 50507f82e..46ded0346 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -696,6 +696,17 @@ body {
 }
 .chart-zoom-reset:hover { color: var(--text-strong); border-color: var(--accent); }
 
+.chart-dir-btn {
+  flex-shrink: 0;
+  font-size: 11px;
+  padding: 1px 5px;
+  font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+  opacity: 0.35;
+  transition: opacity 0.15s;
+}
+.chart-panel:hover .chart-dir-btn { opacity: 1; }
+.chart-dir-btn.active { color: var(--accent); border-color: var(--accent); background: var(--accent-bg); opacity: 1; }
+
 /* ---- chart annotations ------------------------------------------------ */
 .chart-annot {
   margin-top: 6px;
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index f08d7a16f..0ed684f05 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -41,6 +41,7 @@
     tableHideSame: false,
     combineSelection: new Set(),  // metric names currently selected for combining
     combinedCharts: [],           // [{id, metrics: string[]}]
+    metricDir: {},                // metric name -> true (lower better) | false (higher better); absent = auto
   };
 
   // setInterval handles for log auto-refresh; null when not running.
@@ -119,6 +120,7 @@
       activeTab: state.activeTab,
       theme: state.theme,
       combinedCharts: state.combinedCharts,
+      metricDir: state.metricDir,
     };
     try { localStorage.setItem('spt-web-state', JSON.stringify(snap)); } catch {}
     const ids = [...state.visible].map(id => encodeURIComponent(id)).join(',');
@@ -166,6 +168,7 @@
       if (typeof saved.activeTab === 'string') state.activeTab = saved.activeTab;
       if (typeof saved.theme === 'string') state.theme = saved.theme;
       if (Array.isArray(saved.combinedCharts)) state.combinedCharts = saved.combinedCharts;
+      if (saved.metricDir && typeof saved.metricDir === 'object' && !Array.isArray(saved.metricDir)) state.metricDir = saved.metricDir;
       if (Array.isArray(saved.visible)) state._pendingVisible = new Set(saved.visible);
     }
 
@@ -1368,6 +1371,15 @@
     }, 'image/png');
   }
 
+  function _refreshDirBtn(btn, name) {
+    const isOverride = name in state.metricDir;
+    const lower = isLowerBetter(name);
+    btn.textContent = lower ? '↓ min' : '↑ max';
+    btn.title = (lower ? 'lower is better' : 'higher is better')
+      + (isOverride ? ' (manual — click to toggle)' : ' (auto — click to override)');
+    btn.classList.toggle('active', isOverride);
+  }
+
   function makePanel(fullName, displayName) {
     const panel = document.createElement('div');
     panel.className = 'chart-panel';
@@ -1378,6 +1390,18 @@
     span.textContent = displayName || fullName;
     span.title = fullName;
     title.appendChild(span);
+    const dirBtn = document.createElement('button');
+    dirBtn.type = 'button';
+    dirBtn.className = 'chart-dir-btn icon-btn';
+    _refreshDirBtn(dirBtn, fullName);
+    dirBtn.addEventListener('click', (e) => {
+      e.stopPropagation();
+      state.metricDir[fullName] = !isLowerBetter(fullName);
+      _refreshDirBtn(dirBtn, fullName);
+      scheduleRerender();
+      saveState();
+    });
+    title.appendChild(dirBtn);
     const toggleBtn = document.createElement('button');
     toggleBtn.className = 'chart-combine-toggle icon-btn';
     toggleBtn.type = 'button';
@@ -1708,7 +1732,7 @@
         spanGaps: true,
       });
     }
-    const lower = lowerIsBetter(name);
+    const lower = isLowerBetter(name);
     const seriesStats = series.map(s => {
       const raw = s.ys.filter(v => v != null && isFinite(v));
       const lastVal = raw.length ? raw[raw.length - 1] : null;
@@ -1723,6 +1747,11 @@
     return n.includes('loss') || n.includes('err') || n.includes('perplexity') || n.includes('ppl');
   }
 
+  function isLowerBetter(name) {
+    if (name in state.metricDir) return state.metricDir[name];
+    return lowerIsBetter(name);
+  }
+
   // ---- combine-metrics feature ------------------------------------------
 
   function updateCombineSelectionUI() {
@@ -2062,7 +2091,7 @@
     if (!annot) return;
     if (!seriesStats || seriesStats.length === 0) { annot.replaceChildren(); return; }
 
-    const lower = lowerIsBetter(name);
+    const lower = isLowerBetter(name);
     const validBests = seriesStats.map(s => s.bestVal).filter(v => v != null);
     const overallBest = validBests.length
       ? (lower ? Math.min(...validBests) : Math.max(...validBests))

From d4a8c335af02af766dba50d0b5600b38fec26680 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 13:56:09 -0400
Subject: [PATCH 21/31] added graceful failure when files are deleted

---
 stable_pretraining/web/assets/app.css    | 40 ++++++++++++++++++++++++
 stable_pretraining/web/assets/app.js     | 39 +++++++++++++++++++++++
 stable_pretraining/web/assets/index.html |  1 +
 stable_pretraining/web/scan.py           | 20 ++++++++++--
 4 files changed, 97 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.css b/stable_pretraining/web/assets/app.css
index 46ded0346..ce9cf953f 100644
--- a/stable_pretraining/web/assets/app.css
+++ b/stable_pretraining/web/assets/app.css
@@ -1286,3 +1286,43 @@ kbd {
 img.media-content {
   cursor: zoom-in;
 }
+
+/* ---- toast notifications -------------------------------------------- */
+#toast-container {
+  position: fixed;
+  top: 56px;
+  right: 14px;
+  z-index: 600;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+  pointer-events: none;
+  max-width: 340px;
+}
+.toast {
+  background: var(--surface);
+  border: 1px solid var(--border-strong);
+  border-left: 3px solid var(--warn);
+  border-radius: 6px;
+  padding: 10px 12px;
+  font-size: 12px;
+  color: var(--text);
+  box-shadow: var(--shadow);
+  display: flex;
+  align-items: flex-start;
+  gap: 10px;
+  pointer-events: all;
+  animation: toast-in 0.18s ease-out;
+}
+.toast-msg { flex: 1; min-width: 0; line-height: 1.45; word-break: break-word; }
+.toast-close {
+  background: none; border: none;
+  color: var(--muted-2); font-size: 18px; line-height: 1;
+  cursor: pointer; padding: 0; flex-shrink: 0;
+  font-family: inherit;
+}
+.toast-close:hover { color: var(--text); }
+@keyframes toast-in {
+  from { opacity: 0; transform: translateX(12px); }
+  to   { opacity: 1; transform: translateX(0); }
+}
diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 0ed684f05..59ccef9e5 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -3036,6 +3036,15 @@
       try { payload = JSON.parse(ev.data); } catch { return; }
       const { changed = [], removed = [] } = payload;
 
+      // Capture names + visibility BEFORE refreshRuns() wipes them from state.
+      const removedNotable = removed.filter(id =>
+        state.visible.has(id) || id === state.detailRunId
+      );
+      const removedNames = new Map(removedNotable.map(id => {
+        const r = state.runs.get(id);
+        return [id, (r && r.display_name) || id.split('/').pop() || id];
+      }));
+
       // Refresh the index (cheap; sidecars only).
       await refreshRuns();
 
@@ -3071,12 +3080,42 @@
       if (state.activeTab === 'out' || state.activeTab === 'err') {
         _updateLiveState(state.activeTab);
       }
+      // Notify the user for any selected/viewed runs that were deleted.
+      for (const id of removedNotable) {
+        if (!state.runs.has(id)) {
+          showToast(`Run "${removedNames.get(id)}" was deleted from disk.`);
+        }
+      }
     });
     es.addEventListener('error', () => {
       // Browser auto-reconnects.
     });
   }
 
+  // ---- toast notifications ----------------------------------------------
+
+  function showToast(msg, ttl = 7000) {
+    const container = document.getElementById('toast-container');
+    if (!container) return;
+    const toast = document.createElement('div');
+    toast.className = 'toast';
+    const msgEl = document.createElement('span');
+    msgEl.className = 'toast-msg';
+    msgEl.textContent = msg;
+    const closeBtn = document.createElement('button');
+    closeBtn.type = 'button';
+    closeBtn.className = 'toast-close';
+    closeBtn.textContent = '×';
+    closeBtn.title = 'dismiss';
+    toast.appendChild(msgEl);
+    toast.appendChild(closeBtn);
+    container.appendChild(toast);
+    let timer;
+    const dismiss = () => { clearTimeout(timer); toast.remove(); };
+    closeBtn.addEventListener('click', dismiss);
+    timer = setTimeout(dismiss, ttl);
+  }
+
   // ---- controls ---------------------------------------------------------
 
   function debounce(fn, ms) {
diff --git a/stable_pretraining/web/assets/index.html b/stable_pretraining/web/assets/index.html
index 739046d87..76062c6cf 100644
--- a/stable_pretraining/web/assets/index.html
+++ b/stable_pretraining/web/assets/index.html
@@ -151,6 +151,7 @@
     </table>
   </div>
 </div>
+<div id="toast-container" aria-live="polite" aria-atomic="false"></div>
 <script src="https://cdn.jsdelivr.net/npm/uplot@1.6.31/dist/uPlot.iife.min.js"></script>
 <script src="/assets/app.js"></script>
 </body>
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index 934987e91..8fa754728 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -438,7 +438,11 @@ def metrics_json(self, run_id: str) -> Optional[dict]:
             if cached is not None and (cached[0], cached[1]) == cache_key:
                 return cached[2]  # parsed dict (HTTP layer prefers metrics_json_bytes)
 
-        with mpath.open("r", newline="") as f:
+        try:
+            fh = mpath.open("r", newline="")
+        except OSError:
+            return {"metrics": {}}
+        with fh as f:
             reader = csv.reader(f)
             try:
                 header = next(reader)
@@ -560,7 +564,13 @@ def metrics_stream(self, run_id: str):
 
         # Cold path: parse + emit incrementally so the browser can paint
         # while the file is still being read.
-        with mpath.open("r", newline="") as f:
+        try:
+            fh = mpath.open("r", newline="")
+        except OSError:
+            yield {"chunk": 0, "metrics": {}}
+            yield {"done": True}
+            return
+        with fh as f:
             reader = csv.reader(f)
             try:
                 header = next(reader)
@@ -870,7 +880,11 @@ def _find_path(paths_by_label: dict[str, Path]) -> Optional[Path]:
             size = path.stat().st_size
         except OSError:
             return None
-        with path.open("rb") as f:
+        try:
+            fh = path.open("rb")
+        except OSError:
+            return None
+        with fh as f:
             if size > max_bytes:
                 f.seek(size - max_bytes)
                 # Discard up to the next newline to avoid a truncated line.

From 076950688725e2dcb3264b6f68b13012c758c101 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 14:11:52 -0400
Subject: [PATCH 22/31] added more graceful error catching to renaming

---
 stable_pretraining/web/assets/app.js | 12 +++++++++---
 stable_pretraining/web/scan.py       |  8 ++++++++
 stable_pretraining/web/server.py     |  3 +++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index 59ccef9e5..c58f3b911 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -753,9 +753,11 @@
           displayEl.textContent = val;
           const run = state.runs.get(r.run_id);
           if (run) run.display_name = val;
-          patchRunMeta(r.run_id, { display_name: val }).catch(() => {
+          patchRunMeta(r.run_id, { display_name: val }).catch(err => {
             displayEl.textContent = prev;
-            if (run) run.display_name = prev;
+            const cur = state.runs.get(r.run_id);
+            if (cur) cur.display_name = prev;
+            showToast(`Rename failed — ${err.message || 'server error'}`);
           });
         }
       }
@@ -866,6 +868,10 @@
     }
     // Ungrouped runs (or all runs when groupBy is empty) flat at the bottom.
     for (const r of ungrouped) frag.appendChild(makeRunRow(r));
+    // Commit any in-progress inline name edit before tearing out the DOM —
+    // Chrome doesn't fire blur when an element is removed from the tree.
+    const activeEdit = el.querySelector('.run-name-edit');
+    if (activeEdit) activeEdit.blur();
     el.replaceChildren(frag);
   }
 
@@ -2914,8 +2920,8 @@
       try {
         await patchRunMeta(run.run_id, { notes: newVal });
       } catch (e) {
-        console.warn('notes patch failed', e);
         ta.value = run.notes = oldVal;
+        showToast(`Notes save failed — ${e.message || 'server error'}`);
       }
     });
     sec.appendChild(h);
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index 8fa754728..f56686f38 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -760,6 +760,14 @@ def patch_run_meta(self, run_id: str, patch: dict) -> bool:
         bad = set(patch) - self._MUTABLE_FIELDS
         if bad:
             raise ValueError(f"unknown patch fields: {sorted(bad)}")
+        if "display_name" in patch and not isinstance(patch["display_name"], (str, type(None))):
+            raise ValueError("display_name must be a string")
+        if "notes" in patch and not isinstance(patch["notes"], (str, type(None))):
+            raise ValueError("notes must be a string")
+        if "tags" in patch and not isinstance(patch["tags"], list):
+            raise ValueError("tags must be a list")
+        if "archived" in patch and not isinstance(patch["archived"], bool):
+            raise ValueError("archived must be a boolean")
 
         with self._lock:
             run = self._runs.get(run_id)
diff --git a/stable_pretraining/web/server.py b/stable_pretraining/web/server.py
index b3cfb5da2..242e1015a 100644
--- a/stable_pretraining/web/server.py
+++ b/stable_pretraining/web/server.py
@@ -191,6 +191,9 @@ def do_PATCH(self) -> None:  # noqa: N802
             except ValueError as exc:
                 self._serve_json({"error": str(exc)}, 400)
                 return
+            except OSError as exc:
+                self._serve_json({"error": f"write failed: {exc}"}, 500)
+                return
 
             if not found:
                 self._serve_json({"error": "run not found"}, 404)

From 7dfc150d3c20fb08738afdedecf562e019789c9d Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 14:27:45 -0400
Subject: [PATCH 23/31] updates for agents to understand how to interact with
 spt web

---
 AGENTS.md | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)

diff --git a/AGENTS.md b/AGENTS.md
index 0d73a62af..25f433961 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -19,6 +19,7 @@ stable_pretraining/
   data/             # HFDataset, MultiViewTransform, RepeatedRandomSampler
   loggers/          # WandB, Trackio, SwanLab integrations
   registry/         # filesystem-first run registry (sidecars + SQLite)
+  web/              # local run viewer (spt web); reads RegistryLogger output
   optim/            # optimizer and scheduler factories
   utils/            # atomic checkpointing, lightning patch, error handling
   _config.py        # global config: spt.set(key, value) / spt.get_config()
@@ -224,6 +225,118 @@ spt registry export sweep.csv    # export to CSV
 spt registry scan --full         # rebuild SQLite cache
 ```
 
+## spt web — local run viewer
+
+`spt web` is a dependency-free local viewer for `RegistryLogger` runs — a
+WandB alternative requiring no account, no internet, and no extra dependencies
+beyond what the library ships. It reads `sidecar.json` + `metrics.csv` produced
+by `RegistryLogger` and serves a browser UI over stdlib `http.server` with
+Server-Sent Events for live updates.
+
+### Launching
+
+```bash
+# Scan any directory (scanner walks the tree to any depth)
+spt web runs/
+
+# No argument → uses {cache_dir}/runs automatically
+spt web
+
+# Custom host / port / poll interval (mtime-based, no inotify — NFS-safe)
+spt web runs/ --host 0.0.0.0 --port 8080 --poll 2.0
+```
+
+The viewer opens at `http://127.0.0.1:4242` by default. The page updates in
+real time as training writes new metrics — no reload needed.
+
+### Connection to RegistryLogger
+
+`Manager` automatically injects a `RegistryLogger` into every run. When using
+`Manager`, no extra configuration is needed: `spt web {cache_dir}/runs` will
+show all runs. The logger writes to `{run_dir}/`:
+
+| File | Written by | Displayed as |
+|------|------------|-------------|
+| `sidecar.json` | RegistryLogger | run metadata, hparams, summary, status, tags, notes |
+| `metrics.csv` | RegistryLogger | per-step charts (figures tab) |
+| `heartbeat` | RegistryLogger | mtime → stale detection (>5 min old without terminal status = ⚠ stale) |
+| `media.jsonl` | RegistryLogger | image/video media panel |
+| `*.out` / `*.err` | training process | log tab (auto-tails live runs) |
+
+To use `RegistryLogger` directly (outside `Manager`):
+```python
+from stable_pretraining.registry import RegistryLogger
+logger = RegistryLogger(run_dir="runs/my_run", run_id="my_run")
+trainer = pl.Trainer(logger=logger, ...)
+```
+
+### Run directory layout (what spt web expects)
+
+```
+{run_dir}/               ← any leaf directory that contains a sidecar.json
+  sidecar.json           ← required: run metadata
+  metrics.csv            ← optional: columns = step, epoch, <metric_name>, ...
+  heartbeat              ← optional: empty file; mtime = last alive timestamp
+  media.jsonl            ← optional: one JSON object per line, image/video events
+  train.out / train.err  ← optional: log files shown in the .out / .err tabs
+  checkpoints/           ← optional: ignored by the viewer
+```
+
+Runs do not need to be at the top level of the scanned directory — the scanner
+recurses to any depth.
+
+### sidecar.json schema
+
+Key fields agents may read or patch:
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `run_id` | `str` | Unique identifier, usually path relative to cache_dir |
+| `display_name` | `str` | Human-readable label shown in the sidebar (editable via UI or `PATCH /api/run-meta`) |
+| `status` | `str` | `"running"` \| `"completed"` \| `"failed"` \| `"orphaned"` |
+| `hparams` | `dict` | Hyperparameters; logged via `log_hyperparams` |
+| `summary` | `dict` | Final scalars (e.g. best val_acc); populated by `RegistryLogger` at `finalize` |
+| `tags` | `list[str]` | Labels for filtering and grouping (editable via UI) |
+| `notes` | `str` | Free-text notes (editable via UI) |
+| `created_at` | `float` | Unix timestamp of run start |
+| `ended_at` | `float \| None` | Unix timestamp of run end; `None` while still running |
+
+The sidecar can be patched programmatically via the server's HTTP API while
+`spt web` is running:
+```python
+import requests
+requests.patch("http://127.0.0.1:4242/api/run-meta", json={
+    "run_id": "runs/my_run",
+    "display_name": "experiment-v2",
+    "tags": ["sweep", "lr-1e-3"],
+    "notes": "Increased weight decay.",
+})
+```
+Allowed mutable fields: `display_name`, `notes`, `tags`, `archived`.
+
+### UI features (for agents reasoning about what users can do)
+
+| Tab / panel | What it shows | Key interactions |
+|-------------|--------------|-----------------|
+| **Figures** | One uPlot chart per metric, all visible runs overlaid | Drag to zoom (synced across all charts); ↓ min / ↑ max direction toggle; `+` to combine metrics into one panel; `⬇` to download as PNG |
+| **Table** | Runs × (hparams + summary) comparison grid | Sortable columns; column search; "hide same" collapses identical columns; amber diff highlighting |
+| **.out / .err** | Last ~4 MiB of log files | Auto-refreshes every 10 s while run is live; pause button stops auto-refresh |
+| **Detail modal** | Full hparams / summary / tags / notes for one run | Notes and tags are editable in-place; double-click run name in sidebar to rename |
+| **Sidebar** | Run list | Search by name/tag/hparam; filter by field value; group-by any hparam key; sort by any metric |
+
+State (selected runs, filters, active tab, smoothing, theme) persists to
+`localStorage` across reloads. Visible run IDs and active tab are also written
+into the URL fragment (`#runs=id1,id2&tab=figures`) so a shared URL restores the
+exact selection.
+
+### When to suggest `spt web`
+
+- User wants to inspect or compare runs **locally** (no WandB account needed).
+- User wants to **monitor a live training run** — the viewer updates as metrics arrive.
+- User is running a **hyperparameter sweep** and wants to compare configs (table tab + hide-same).
+- User wants to see **log output** from a run: `spt web {parent_dir}`, then open the .out tab.
+- User asks "how do I see my training curves?" — `spt web` is the answer unless they already have WandB configured.
+
 ## Callback ordering
 
 Lightning runs `trainer.callbacks` in registration order. Within a single hook, callbacks fire in that order; across hooks, Lightning completes each hook for **every** callback before moving to the next.

From e55a34c7dba2f738cfb5450d450c9dc9a551f708 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 14:50:09 -0400
Subject: [PATCH 24/31] added lots of unit tests

---
 stable_pretraining/tests/unit/test_web.py | 1033 +++++++++++++++++++++
 1 file changed, 1033 insertions(+)
 create mode 100644 stable_pretraining/tests/unit/test_web.py

diff --git a/stable_pretraining/tests/unit/test_web.py b/stable_pretraining/tests/unit/test_web.py
new file mode 100644
index 000000000..2063ec44b
--- /dev/null
+++ b/stable_pretraining/tests/unit/test_web.py
@@ -0,0 +1,1033 @@
+"""Unit tests for the spt web viewer backend.
+
+
+  File-deletion safety
+    · metrics_json  returns {"metrics": {}} when metrics.csv is missing or
+      raises OSError on open (simulates file deleted mid-request)
+    · metrics_stream yields a "done" chunk instead of crashing
+    · log_content   returns None instead of crashing
+
+  Rename / edit robustness
+    · patch_run_meta raises ValueError for unknown keys and wrong value types
+    · patch_run_meta returns False for an unknown run_id
+    · patch_run_meta happy path persists changes to the sidecar on disk
+    · do_PATCH HTTP handler returns 400 for bad fields / types, 404 for
+      unknown runs, 500 when write_sidecar raises OSError, and 200 on success
+
+  Serialization and data shapes
+    · _serialize display_name resolution order
+    · runs_json correct list structure and fields
+    · metrics_json_bytes byte cache and NaN/Inf sanitization
+    · _safe_dumps NaN/Inf → null
+    · CSV edge cases (header-only, non-finite values)
+    · log_content truncation behaviour
+    · logs_index discovery and stream_id stability
+
+  HTTP GET endpoints
+    · All GET routes (/, /assets, /api/runs, /api/scan-status, /api/metrics,
+      /api/logs, /api/log-content) return expected status codes
+    · Path traversal blocked with 403
+    · Missing parameters return 400
+"""
+
+from __future__ import annotations
+
+import http.client
+import json
+import math
+import threading
+import urllib.error
+import urllib.request
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from stable_pretraining.web.scan import RunScanner, _Run, _safe_log_id
+from stable_pretraining.web.server import _Handler, _safe_dumps
+
+pytestmark = pytest.mark.unit
+
+
+# ── shared helpers ────────────────────────────────────────────────────────────
+
+
+def _make_scanner(root: Path) -> RunScanner:
+    """Return a scanner with no background thread (poll_interval is irrelevant)."""
+    return RunScanner(root, poll_interval=999)
+
+
+def _inject_run(scanner: RunScanner, run_dir: Path, **sidecar_kw) -> str:
+    """Write a sidecar, create the run directory, and register the run in
+    the scanner's in-memory map without starting the background thread."""
+    from stable_pretraining.registry._sidecar import make_sidecar, write_sidecar
+
+    run_dir.mkdir(parents=True, exist_ok=True)
+    run_id = str(run_dir.relative_to(scanner.root))
+    sc = make_sidecar(run_id=run_id, run_dir=str(run_dir), **sidecar_kw)
+    write_sidecar(run_dir, sc)
+    run = _Run(
+        run_id=run_id,
+        run_dir=run_dir,
+        sidecar_mtime=0.0,
+        metrics_mtime=0.0,
+        metrics_size=0,
+        sidecar=sc,
+    )
+    with scanner._lock:
+        scanner._runs[run_id] = run
+    return run_id
+
+
+@pytest.fixture
+def web_server(tmp_path):
+    """Spin up a real ThreadingHTTPServer on a free port.
+
+    Yields ``(scanner, base_url)`` and tears down cleanly after the test.
+    """
+    sc = _make_scanner(tmp_path)
+
+    class Handler(_Handler):
+        pass
+
+    Handler.scanner = sc
+
+    srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    port = srv.server_address[1]
+    t = threading.Thread(target=srv.serve_forever, daemon=True)
+    t.start()
+    yield sc, f"http://127.0.0.1:{port}"
+    srv.shutdown()
+    srv.server_close()
+
+
+def _do_patch(base_url: str, payload: dict) -> tuple[int, dict]:
+    """Send a PATCH /api/run-meta and return (status_code, json_body)."""
+    body = json.dumps(payload).encode()
+    req = urllib.request.Request(
+        base_url + "/api/run-meta",
+        data=body,
+        method="PATCH",
+        headers={
+            "Content-Type": "application/json",
+            "Content-Length": str(len(body)),
+        },
+    )
+    try:
+        with urllib.request.urlopen(req) as resp:
+            return resp.status, json.loads(resp.read())
+    except urllib.error.HTTPError as exc:
+        return exc.code, json.loads(exc.read())
+
+
+# ── TOCTOU safety ───────────────────────────────────────────────────────
+
+
+class TestTOCTOUSafety:
+    """Deleting files mid-request must never raise unhandled exceptions."""
+
+    # ---- metrics_json --------------------------------------------------------
+
+    def test_metrics_json_unknown_run_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.metrics_json("nonexistent") is None
+
+    def test_metrics_json_missing_file_returns_empty(self, tmp_path):
+        """metrics.csv never written — simulates deletion before request arrives."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        result = sc.metrics_json(run_id)
+        assert result == {"metrics": {}}
+
+    def test_metrics_json_oserror_on_open_returns_empty(self, tmp_path):
+        """TOCTOU: is_file() passes, open() raises OSError.
+
+        Verifies the try/except OSError guard around mpath.open() in
+        metrics_json.
+        """
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        (tmp_path / "r1" / "metrics.csv").write_text("step,loss\n1,0.5\n")
+
+        original_open = Path.open
+
+        def _raiser(self, *args, **kwargs):
+            if self.name == "metrics.csv":
+                raise FileNotFoundError("deleted mid-request")
+            return original_open(self, *args, **kwargs)
+
+        with patch.object(Path, "open", _raiser):
+            result = sc.metrics_json(run_id)
+
+        assert result == {"metrics": {}}
+
+    def test_metrics_json_parses_valid_csv(self, tmp_path):
+        """Sanity check: a real CSV is parsed correctly."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        (tmp_path / "r1" / "metrics.csv").write_text(
+            "step,epoch,train/loss\n0,0,2.3\n10,0,2.1\n20,1,1.9\n"
+        )
+        result = sc.metrics_json(run_id)
+        assert "train/loss" in result["metrics"]
+        assert len(result["metrics"]["train/loss"]["y"]) == 3
+
+    # ---- metrics_stream ------------------------------------------------------
+
+    def test_metrics_stream_unknown_run_yields_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        chunks = list(sc.metrics_stream("nonexistent"))
+        assert chunks == [None]
+
+    def test_metrics_stream_missing_file_yields_done(self, tmp_path):
+        """metrics.csv absent: stream must end cleanly."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        chunks = [c for c in sc.metrics_stream(run_id) if c is not None]
+        assert any(c.get("done") for c in chunks)
+        metrics_chunks = [c for c in chunks if "metrics" in c]
+        assert all(c["metrics"] == {} for c in metrics_chunks)
+
+    def test_metrics_stream_oserror_on_open_yields_done(self, tmp_path):
+        """TOCTOU: is_file() passes, open() raises OSError.
+
+        Verifies the try/except OSError guard around mpath.open() in
+        metrics_stream cold-path.
+        """
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        (tmp_path / "r1" / "metrics.csv").write_text("step,loss\n1,0.5\n")
+
+        original_open = Path.open
+
+        def _raiser(self, *args, **kwargs):
+            if self.name == "metrics.csv":
+                raise FileNotFoundError("deleted mid-request")
+            return original_open(self, *args, **kwargs)
+
+        with patch.object(Path, "open", _raiser):
+            chunks = [c for c in sc.metrics_stream(run_id) if c is not None]
+
+        assert any(c.get("done") for c in chunks)
+        metrics_chunks = [c for c in chunks if "metrics" in c]
+        assert all(c["metrics"] == {} for c in metrics_chunks)
+
+    def test_metrics_stream_yields_data_then_done(self, tmp_path):
+        """Sanity check: a real CSV streams data and ends with done."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        (tmp_path / "r1" / "metrics.csv").write_text(
+            "step,val/acc\n" + "\n".join(f"{i},{i*0.01:.3f}" for i in range(20))
+        )
+        chunks = [c for c in sc.metrics_stream(run_id) if c is not None]
+        assert any(c.get("done") for c in chunks)
+        all_y = []
+        for c in chunks:
+            if "metrics" in c and "val/acc" in c["metrics"]:
+                all_y.extend(c["metrics"]["val/acc"]["y"])
+        assert len(all_y) == 20
+
+    # ---- log_content ---------------------------------------------------------
+
+    def test_log_content_unknown_run_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.log_content("nonexistent", "train.out") is None
+
+    def test_log_content_missing_file_returns_none(self, tmp_path):
+        """Stream registered in _log_paths but file absent."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        missing = tmp_path / "r1" / "train.out"
+        sc._log_paths[run_id] = {"train.out": missing}
+        stream_id = _safe_log_id("train.out")
+        result = sc.log_content(run_id, stream_id)
+        assert result is None
+
+    def test_log_content_oserror_on_open_returns_none(self, tmp_path):
+        """TOCTOU: stat() passes, open() raises OSError.
+
+        Verifies the try/except OSError guard around path.open("rb") in
+        log_content.
+        """
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        log_file = tmp_path / "r1" / "train.out"
+        log_file.write_text("epoch 1 loss=2.30\n" * 10)
+        stream_id = _safe_log_id("train.out")
+        sc._log_paths[run_id] = {"train.out": log_file}
+
+        original_open = Path.open
+
+        def _raiser(self, *args, **kwargs):
+            if self.name == "train.out":
+                raise FileNotFoundError("deleted mid-request")
+            return original_open(self, *args, **kwargs)
+
+        with patch.object(Path, "open", _raiser):
+            result = sc.log_content(run_id, stream_id)
+
+        assert result is None
+
+    def test_log_content_reads_existing_file(self, tmp_path):
+        """Sanity check: log_content returns the file bytes."""
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        log_file = tmp_path / "r1" / "train.out"
+        log_file.write_bytes(b"hello log\n")
+        stream_id = _safe_log_id("train.out")
+        sc._log_paths[run_id] = {"train.out": log_file}
+        result = sc.log_content(run_id, stream_id)
+        assert result == b"hello log\n"
+
+
+# ── patch_run_meta validation ──────────────────────────────────────────
+
+
+class TestPatchRunMetaValidation:
+    """patch_run_meta enforces field-name and value-type constraints."""
+
+    def _setup(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        return sc, run_id
+
+    # ---- field-name checks ---------------------------------------------------
+
+    def test_unknown_field_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="unknown patch fields"):
+            sc.patch_run_meta(run_id, {"bad_key": "x"})
+
+    def test_mix_of_valid_and_unknown_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="unknown patch fields"):
+            sc.patch_run_meta(run_id, {"display_name": "ok", "bad_key": "x"})
+
+    # ---- display_name type checks --------------------------------------------
+
+    def test_display_name_int_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="display_name"):
+            sc.patch_run_meta(run_id, {"display_name": 123})
+
+    def test_display_name_list_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="display_name"):
+            sc.patch_run_meta(run_id, {"display_name": ["name"]})
+
+    def test_display_name_bool_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="display_name"):
+            sc.patch_run_meta(run_id, {"display_name": True})
+
+    def test_display_name_string_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"display_name": "my-run"}) is True
+
+    def test_display_name_none_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"display_name": None}) is True
+
+    # ---- notes type checks ---------------------------------------------------
+
+    def test_notes_int_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="notes"):
+            sc.patch_run_meta(run_id, {"notes": 99})
+
+    def test_notes_list_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="notes"):
+            sc.patch_run_meta(run_id, {"notes": []})
+
+    def test_notes_string_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"notes": "a note"}) is True
+
+    def test_notes_none_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"notes": None}) is True
+
+    # ---- tags type checks ----------------------------------------------------
+
+    def test_tags_string_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="tags"):
+            sc.patch_run_meta(run_id, {"tags": "not-a-list"})
+
+    def test_tags_none_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="tags"):
+            sc.patch_run_meta(run_id, {"tags": None})
+
+    def test_tags_list_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"tags": ["sweep", "lr-1e-3"]}) is True
+
+    def test_tags_empty_list_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"tags": []}) is True
+
+    # ---- archived type checks ------------------------------------------------
+
+    def test_archived_string_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="archived"):
+            sc.patch_run_meta(run_id, {"archived": "yes"})
+
+    def test_archived_int_raises(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        with pytest.raises(ValueError, match="archived"):
+            sc.patch_run_meta(run_id, {"archived": 1})
+
+    def test_archived_bool_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        assert sc.patch_run_meta(run_id, {"archived": True}) is True
+
+    # ---- other edge cases ----------------------------------------------------
+
+    def test_unknown_run_id_returns_false(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.patch_run_meta("does-not-exist", {"display_name": "x"}) is False
+
+    def test_all_valid_fields_accepted(self, tmp_path):
+        sc, run_id = self._setup(tmp_path)
+        result = sc.patch_run_meta(
+            run_id,
+            {
+                "display_name": "experiment-v2",
+                "notes": "increased weight decay",
+                "tags": ["sweep", "wd-1e-2"],
+                "archived": False,
+            },
+        )
+        assert result is True
+
+    def test_patch_persists_display_name_to_sidecar(self, tmp_path):
+        """The new display_name must survive a sidecar re-read."""
+        from stable_pretraining.registry._sidecar import read_sidecar, sidecar_path
+
+        sc, run_id = self._setup(tmp_path)
+        sc.patch_run_meta(run_id, {"display_name": "renamed"})
+        run_dir = tmp_path / "r1"
+        on_disk = read_sidecar(sidecar_path(run_dir))
+        assert on_disk["display_name"] == "renamed"
+
+    def test_patch_persists_tags_to_sidecar(self, tmp_path):
+        from stable_pretraining.registry._sidecar import read_sidecar, sidecar_path
+
+        sc, run_id = self._setup(tmp_path)
+        sc.patch_run_meta(run_id, {"tags": ["a", "b"]})
+        on_disk = read_sidecar(sidecar_path(tmp_path / "r1"))
+        assert on_disk["tags"] == ["a", "b"]
+
+    def test_patch_updates_in_memory_sidecar(self, tmp_path):
+        """In-memory run.sidecar must reflect the patch immediately."""
+        sc, run_id = self._setup(tmp_path)
+        sc.patch_run_meta(run_id, {"display_name": "live-update"})
+        with sc._lock:
+            run = sc._runs[run_id]
+        assert run.sidecar["display_name"] == "live-update"
+
+
+# ── HTTP PATCH handler ─────────────────────────────────────────────────
+
+
+class TestPatchHTTPHandler:
+    """do_PATCH must return clean JSON for every outcome."""
+
+    def test_happy_path_returns_200(self, tmp_path, web_server):
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r1")
+        status, body = _do_patch(base, {"run_id": run_id, "display_name": "new-name"})
+        assert status == 200
+        assert body == {"ok": True}
+
+    def test_display_name_persisted_after_patch(self, tmp_path, web_server):
+        from stable_pretraining.registry._sidecar import read_sidecar, sidecar_path
+
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r2")
+        _do_patch(base, {"run_id": run_id, "display_name": "via-http"})
+        on_disk = read_sidecar(sidecar_path(tmp_path / "r2"))
+        assert on_disk["display_name"] == "via-http"
+
+    def test_unknown_run_returns_404(self, web_server):
+        _, base = web_server
+        status, body = _do_patch(base, {"run_id": "no-such-run", "display_name": "x"})
+        assert status == 404
+        assert "error" in body
+
+    def test_missing_run_id_returns_400(self, web_server):
+        _, base = web_server
+        status, body = _do_patch(base, {"display_name": "x"})
+        assert status == 400
+        assert "error" in body
+
+    def test_unknown_field_returns_400(self, tmp_path, web_server):
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r3")
+        status, body = _do_patch(base, {"run_id": run_id, "bad_field": "x"})
+        assert status == 400
+        assert "error" in body
+
+    def test_wrong_type_display_name_returns_400(self, tmp_path, web_server):
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r4")
+        status, body = _do_patch(base, {"run_id": run_id, "display_name": 999})
+        assert status == 400
+        assert "error" in body
+
+    def test_wrong_type_tags_returns_400(self, tmp_path, web_server):
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r5")
+        status, body = _do_patch(base, {"run_id": run_id, "tags": "not-a-list"})
+        assert status == 400
+        assert "error" in body
+
+    def test_oserror_from_write_returns_500(self, tmp_path, web_server):
+        """write_sidecar raising OSError must not crash the thread — returns 500."""
+        scanner, base = web_server
+        run_id = _inject_run(scanner, tmp_path / "r6")
+        with patch(
+            "stable_pretraining.web.scan.write_sidecar",
+            side_effect=OSError("simulated disk error"),
+        ):
+            status, body = _do_patch(base, {"run_id": run_id, "display_name": "x"})
+        assert status == 500
+        assert "error" in body
+
+    def test_invalid_json_body_returns_400(self, web_server):
+        _, base = web_server
+        req = urllib.request.Request(
+            base + "/api/run-meta",
+            data=b"{not valid json",
+            method="PATCH",
+            headers={"Content-Type": "application/json", "Content-Length": "15"},
+        )
+        try:
+            with urllib.request.urlopen(req) as resp:
+                status = resp.status
+        except urllib.error.HTTPError as exc:
+            status = exc.code
+        assert status == 400
+
+    def test_wrong_path_returns_404(self, web_server):
+        _, base = web_server
+        req = urllib.request.Request(
+            base + "/api/no-such-endpoint",
+            data=b"{}",
+            method="PATCH",
+            headers={"Content-Type": "application/json", "Content-Length": "2"},
+        )
+        try:
+            with urllib.request.urlopen(req):
+                pass
+        except urllib.error.HTTPError as exc:
+            assert exc.code == 404
+
+
+# ── GET helpers ───────────────────────────────────────────────────────────────
+
+
+def _do_get(base_url: str, path: str) -> tuple[int, bytes, str]:
+    """GET *path* and return (status, body, content-type)."""
+    try:
+        with urllib.request.urlopen(base_url + path) as resp:
+            return resp.status, resp.read(), resp.headers.get("Content-Type", "")
+    except urllib.error.HTTPError as exc:
+        return exc.code, exc.read(), exc.headers.get("Content-Type", "")
+
+
+def _raw_get(base_url: str, path: str) -> int:
+    """GET with no URL normalisation — needed for path-traversal tests."""
+    port = int(base_url.rsplit(":", 1)[-1])
+    conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
+    conn.request("GET", path)
+    resp = conn.getresponse()
+    status = resp.status
+    resp.read()
+    conn.close()
+    return status
+
+
+# ── TestSafeDumps ─────────────────────────────────────────────────────────────
+
+
+class TestSafeDumps:
+    def test_nan_becomes_null(self):
+        assert json.loads(_safe_dumps(float("nan"))) is None
+
+    def test_pos_inf_becomes_null(self):
+        assert json.loads(_safe_dumps(float("inf"))) is None
+
+    def test_neg_inf_becomes_null(self):
+        assert json.loads(_safe_dumps(float("-inf"))) is None
+
+    def test_nested_nan_in_list(self):
+        result = json.loads(_safe_dumps({"y": [1.0, float("nan"), 3.0]}))
+        assert result == {"y": [1.0, None, 3.0]}
+
+    def test_nested_inf_in_dict(self):
+        result = json.loads(_safe_dumps({"loss": float("inf")}))
+        assert result == {"loss": None}
+
+    def test_finite_floats_unchanged(self):
+        assert json.loads(_safe_dumps(1.5)) == pytest.approx(1.5)
+
+    def test_integers_unchanged(self):
+        assert json.loads(_safe_dumps(42)) == 42
+
+    def test_nested_dict_and_list(self):
+        obj = {"a": [float("nan"), {"b": float("inf"), "c": 2.0}]}
+        result = json.loads(_safe_dumps(obj))
+        assert result == {"a": [None, {"b": None, "c": 2.0}]}
+
+
+# ── TestSerialize ─────────────────────────────────────────────────────────────
+
+
+class TestSerialize:
+    def test_explicit_display_name_wins(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run1"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        run_id = str(run_dir.relative_to(sc.root))
+        from stable_pretraining.registry._sidecar import make_sidecar, write_sidecar
+
+        sid = make_sidecar(run_id=run_id, run_dir=str(run_dir))
+        sid["display_name"] = "My Run"
+        write_sidecar(run_dir, sid)
+        run = _Run(
+            run_id=run_id,
+            run_dir=run_dir,
+            sidecar_mtime=0.0,
+            metrics_mtime=0.0,
+            metrics_size=0,
+            sidecar=sid,
+        )
+        with sc._lock:
+            sc._runs[run_id] = run
+        runs = sc.runs_json()
+        assert len(runs) == 1
+        assert runs[0]["display_name"] == "My Run"
+
+    def test_falls_back_to_last_path_component(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "parent" / "child_run"
+        run_id = _inject_run(sc, run_dir)
+        runs = sc.runs_json()
+        assert runs[0]["display_name"] == "child_run"
+
+    def test_falls_back_to_full_run_id_when_no_slash(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "flat"
+        run_id = _inject_run(sc, run_dir)
+        runs = sc.runs_json()
+        assert runs[0]["display_name"] == "flat"
+
+    def test_has_media_false_when_empty(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run_nomedia"
+        _inject_run(sc, run_dir)
+        runs = sc.runs_json()
+        assert runs[0]["has_media"] is False
+
+    def test_has_media_true_when_media_size_nonzero(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run_media"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        run_id = str(run_dir.relative_to(sc.root))
+        # Construct _Run directly with a non-zero media_size.
+        from stable_pretraining.registry._sidecar import make_sidecar, write_sidecar
+
+        sid = make_sidecar(run_id=run_id, run_dir=str(run_dir))
+        write_sidecar(run_dir, sid)
+        run = _Run(
+            run_id=run_id,
+            run_dir=run_dir,
+            sidecar_mtime=0.0,
+            metrics_mtime=0.0,
+            metrics_size=0,
+            media_mtime=1.0,
+            media_size=512,
+            sidecar=sid,
+        )
+        with sc._lock:
+            sc._runs[run_id] = run
+        runs = sc.runs_json()
+        assert runs[0]["has_media"] is True
+
+    def test_heartbeat_at_none_when_no_file(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run_nohb"
+        _inject_run(sc, run_dir)
+        runs = sc.runs_json()
+        assert runs[0]["heartbeat_at"] is None
+
+
+# ── TestRunsJson ──────────────────────────────────────────────────────────────
+
+
+class TestRunsJson:
+    def test_empty_when_no_runs(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.runs_json() == []
+
+    def test_returns_expected_keys(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        _inject_run(sc, run_dir, tags=["v1"], notes="hello")
+        runs = sc.runs_json()
+        assert len(runs) == 1
+        r = runs[0]
+        for key in (
+            "run_id",
+            "run_dir",
+            "display_name",
+            "status",
+            "created_at",
+            "ended_at",
+            "heartbeat_at",
+            "tags",
+            "notes",
+            "hparams",
+            "summary",
+            "checkpoint_path",
+            "metrics_size",
+            "has_media",
+        ):
+            assert key in r, f"missing key: {key}"
+
+    def test_tags_and_notes_roundtrip(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        _inject_run(sc, run_dir, tags=["a", "b"], notes="my note")
+        r = sc.runs_json()[0]
+        assert r["tags"] == ["a", "b"]
+        assert r["notes"] == "my note"
+
+
+# ── TestMetricsCache ──────────────────────────────────────────────────────────
+
+
+class TestMetricsCache:
+    def _write_csv(self, path: Path, rows: list[list]) -> None:
+        import csv
+
+        with path.open("w", newline="") as f:
+            csv.writer(f).writerows(rows)
+
+    def test_cache_hit_avoids_reparse(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        run_id = _inject_run(sc, run_dir)
+        csv_path = run_dir / "metrics.csv"
+        self._write_csv(csv_path, [["step", "loss"], ["1", "0.5"], ["2", "0.3"]])
+
+        first = sc.metrics_json(run_id)
+        # Second call should hit cache; patch open to detect if file is re-read.
+        with patch("builtins.open", side_effect=AssertionError("file re-opened")):
+            second = sc.metrics_json(run_id)
+        assert first == second
+
+    def test_cache_invalidated_on_file_change(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        run_id = _inject_run(sc, run_dir)
+        csv_path = run_dir / "metrics.csv"
+        self._write_csv(csv_path, [["step", "loss"], ["1", "0.5"]])
+
+        first = sc.metrics_json(run_id)
+        assert len(first["metrics"]["loss"]["y"]) == 1
+
+        # Overwrite the file; bump mtime by writing fresh content.
+        self._write_csv(
+            csv_path,
+            [["step", "loss"], ["1", "0.5"], ["2", "0.3"], ["3", "0.1"]],
+        )
+        # Touch so the mtime definitely changes (some filesystems have 1 s resolution).
+        import os, time
+
+        os.utime(csv_path, (time.time() + 2, time.time() + 2))
+
+        second = sc.metrics_json(run_id)
+        assert len(second["metrics"]["loss"]["y"]) == 3
+
+
+# ── TestMetricsJsonBytes ──────────────────────────────────────────────────────
+
+
+class TestMetricsJsonBytes:
+    def test_returns_valid_utf8_json(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,0.5\n2,0.3\n")
+        body = sc.metrics_json_bytes(run_id)
+        assert isinstance(body, bytes)
+        parsed = json.loads(body.decode("utf-8"))
+        assert "metrics" in parsed
+
+    def test_nan_inf_serialised_as_null(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,nan\n2,inf\n3,0.5\n")
+        body = sc.metrics_json_bytes(run_id)
+        parsed = json.loads(body.decode("utf-8"))
+        y_vals = parsed["metrics"]["loss"]["y"]
+        assert y_vals == [None, None, pytest.approx(0.5)]
+
+    def test_byte_cache_reused_on_second_call(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r3"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,0.5\n")
+        first = sc.metrics_json_bytes(run_id)
+        second = sc.metrics_json_bytes(run_id)
+        assert first is second  # same bytes object from cache
+
+    def test_unknown_run_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.metrics_json_bytes("no-such-run") is None
+
+
+# ── TestCsvEdgeCases ──────────────────────────────────────────────────────────
+
+
+class TestCsvEdgeCases:
+    def test_header_only_csv_returns_empty_metrics(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n")
+        result = sc.metrics_json(run_id)
+        assert result == {"metrics": {}}
+
+    def test_nan_float_in_csv_does_not_crash(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,nan\n2,inf\n3,-inf\n4,0.2\n")
+        result = sc.metrics_json(run_id)
+        assert "loss" in result["metrics"]
+        y = result["metrics"]["loss"]["y"]
+        assert len(y) == 4
+        assert math.isnan(y[0])
+        assert math.isinf(y[1]) and y[1] > 0
+        assert math.isinf(y[2]) and y[2] < 0
+        assert y[3] == pytest.approx(0.2)
+
+    def test_nan_inf_sanitised_in_bytes(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r3"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,nan\n2,inf\n")
+        body = sc.metrics_json_bytes(run_id)
+        parsed = json.loads(body.decode("utf-8"))
+        assert all(v is None for v in parsed["metrics"]["loss"]["y"])
+
+
+# ── TestLogsIndex ─────────────────────────────────────────────────────────────
+
+
+class TestLogsIndex:
+    def test_unknown_run_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.logs_index("no-such") is None
+
+    def test_discovers_out_and_err_files(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "train.out").write_text("stdout")
+        (run_dir / "train.err").write_text("stderr")
+        result = sc.logs_index(run_id)
+        assert result is not None
+        names = {s["name"] for s in result["streams"]}
+        assert any("train.out" in n or n == "train.out" for n in names)
+        assert any("train.err" in n or n == "train.err" for n in names)
+
+    def test_kind_field_correct(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "job.out").write_text("o")
+        (run_dir / "job.err").write_text("e")
+        result = sc.logs_index(run_id)
+        kinds = {s["name"].split("/")[-1]: s["kind"] for s in result["streams"]}
+        out_kinds = {k: v for k, v in kinds.items() if "out" in k}
+        err_kinds = {k: v for k, v in kinds.items() if "err" in k}
+        assert all(v == "out" for v in out_kinds.values())
+        assert all(v == "err" for v in err_kinds.values())
+
+    def test_stream_id_is_stable_slug(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r3"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "run.out").write_text("x")
+        r1 = sc.logs_index(run_id)
+        r2 = sc.logs_index(run_id)
+        ids1 = {s["stream_id"] for s in r1["streams"]}
+        ids2 = {s["stream_id"] for s in r2["streams"]}
+        assert ids1 == ids2  # stable across calls
+
+
+# ── TestLogContentTruncation ──────────────────────────────────────────────────
+
+
+class TestLogContentTruncation:
+    def test_small_file_returned_in_full(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r"
+        run_id = _inject_run(sc, run_dir)
+        content = b"hello\nworld\n"
+        log_file = run_dir / "job.out"
+        log_file.write_bytes(content)
+        sc.logs_index(run_id)  # populate path map
+        stream_id = _safe_log_id("job.out")
+        result = sc.log_content(run_id, stream_id)
+        assert result == content
+
+    def test_large_file_returns_tail(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r2"
+        run_id = _inject_run(sc, run_dir)
+        # Write 3 × max_bytes worth of lines so truncation is guaranteed.
+        max_bytes = 1024
+        line = b"x" * 63 + b"\n"  # 64 bytes per line
+        lines = line * (max_bytes * 3 // len(line) + 1)
+        log_file = run_dir / "job.out"
+        log_file.write_bytes(lines)
+        sc.logs_index(run_id)
+        stream_id = _safe_log_id("job.out")
+        result = sc.log_content(run_id, stream_id, max_bytes=max_bytes)
+        assert result is not None
+        assert len(result) <= max_bytes
+        # The result should end with the last lines of the file.
+        assert lines.endswith(result)
+
+    def test_unknown_run_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        assert sc.log_content("no-run", "no-stream") is None
+
+    def test_unknown_stream_returns_none(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "r3"
+        run_id = _inject_run(sc, run_dir)
+        sc.logs_index(run_id)
+        assert sc.log_content(run_id, "nonexistent-stream-id") is None
+
+
+# ── TestGetEndpoints ──────────────────────────────────────────────────────────
+
+
+class TestGetEndpoints:
+    def test_root_returns_200_html(self, web_server):
+        _, base = web_server
+        status, body, ctype = _do_get(base, "/")
+        assert status == 200
+        assert "text/html" in ctype
+        assert b"<html" in body.lower() or b"<!doctype" in body.lower()
+
+    def test_assets_app_js_returns_200(self, web_server):
+        _, base = web_server
+        status, body, ctype = _do_get(base, "/assets/app.js")
+        assert status == 200
+        assert len(body) > 0
+
+    def test_unknown_asset_returns_404(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/assets/does-not-exist.xyz")
+        assert status == 404
+
+    def test_path_traversal_returns_403(self, web_server):
+        _, base = web_server
+        status = _raw_get(base, "/assets/../../etc/passwd")
+        assert status == 403
+
+    def test_api_runs_returns_200_json_list(self, web_server):
+        sc, base = web_server
+        run_dir = sc.root / "r"
+        _inject_run(sc, run_dir)
+        status, body, ctype = _do_get(base, "/api/runs")
+        assert status == 200
+        assert "application/json" in ctype
+        parsed = json.loads(body)
+        assert isinstance(parsed, list)
+        assert len(parsed) == 1
+        assert parsed[0]["run_id"] == "r"
+
+    def test_api_scan_status_returns_phase(self, web_server):
+        _, base = web_server
+        status, body, ctype = _do_get(base, "/api/scan-status")
+        assert status == 200
+        parsed = json.loads(body)
+        assert "phase" in parsed
+
+    def test_api_metrics_missing_run_id_returns_400(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/api/metrics")
+        assert status == 400
+
+    def test_api_metrics_unknown_run_returns_404(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/api/metrics?run_id=ghost")
+        assert status == 404
+
+    def test_api_metrics_valid_run_returns_200(self, web_server):
+        sc, base = web_server
+        run_dir = sc.root / "rm"
+        run_id = _inject_run(sc, run_dir)
+        (run_dir / "metrics.csv").write_text("step,loss\n1,0.5\n")
+        status, body, ctype = _do_get(base, f"/api/metrics?run_id={run_id}")
+        assert status == 200
+        parsed = json.loads(body)
+        assert "metrics" in parsed
+
+    def test_api_logs_missing_run_id_returns_400(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/api/logs")
+        assert status == 400
+
+    def test_api_logs_unknown_run_returns_404(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/api/logs?run_id=ghost")
+        assert status == 404
+
+    def test_api_logs_valid_run_returns_streams(self, web_server):
+        sc, base = web_server
+        run_dir = sc.root / "rl"
+        run_id = _inject_run(sc, run_dir)
+        status, body, _ = _do_get(base, f"/api/logs?run_id={run_id}")
+        assert status == 200
+        parsed = json.loads(body)
+        assert "streams" in parsed
+
+    def test_api_log_content_missing_params_returns_400(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/api/log-content")
+        assert status == 400
+
+    def test_api_log_content_valid_returns_200_text(self, web_server):
+        sc, base = web_server
+        run_dir = sc.root / "rc"
+        run_id = _inject_run(sc, run_dir)
+        log_file = run_dir / "job.out"
+        log_file.write_text("hello log")
+        # Populate the path map via logs_index first.
+        sc.logs_index(run_id)
+        stream_id = _safe_log_id("job.out")
+        status, body, ctype = _do_get(
+            base, f"/api/log-content?run_id={run_id}&stream_id={stream_id}"
+        )
+        assert status == 200
+        assert "text/plain" in ctype
+        assert b"hello log" in body
+
+    def test_unknown_path_returns_404(self, web_server):
+        _, base = web_server
+        status, _, _ = _do_get(base, "/no-such-route")
+        assert status == 404

From 2714a7d019264e2cde86e1e4d5595f91514d0f35 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 14:55:33 -0400
Subject: [PATCH 25/31] fixing precommit errors

---
 stable_pretraining/tests/unit/test_web.py | 86 ++++++++++++++---------
 stable_pretraining/web/scan.py            |  4 +-
 2 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/stable_pretraining/tests/unit/test_web.py b/stable_pretraining/tests/unit/test_web.py
index 2063ec44b..39cfbbdf8 100644
--- a/stable_pretraining/tests/unit/test_web.py
+++ b/stable_pretraining/tests/unit/test_web.py
@@ -1,33 +1,32 @@
 """Unit tests for the spt web viewer backend.
 
-
   File-deletion safety
-    · metrics_json  returns {"metrics": {}} when metrics.csv is missing or
-      raises OSError on open (simulates file deleted mid-request)
-    · metrics_stream yields a "done" chunk instead of crashing
-    · log_content   returns None instead of crashing
-
-  Rename / edit robustness
-    · patch_run_meta raises ValueError for unknown keys and wrong value types
-    · patch_run_meta returns False for an unknown run_id
-    · patch_run_meta happy path persists changes to the sidecar on disk
-    · do_PATCH HTTP handler returns 400 for bad fields / types, 404 for
-      unknown runs, 500 when write_sidecar raises OSError, and 200 on success
-
-  Serialization and data shapes
-    · _serialize display_name resolution order
-    · runs_json correct list structure and fields
-    · metrics_json_bytes byte cache and NaN/Inf sanitization
-    · _safe_dumps NaN/Inf → null
-    · CSV edge cases (header-only, non-finite values)
-    · log_content truncation behaviour
-    · logs_index discovery and stream_id stability
-
-  HTTP GET endpoints
-    · All GET routes (/, /assets, /api/runs, /api/scan-status, /api/metrics,
-      /api/logs, /api/log-content) return expected status codes
-    · Path traversal blocked with 403
-    · Missing parameters return 400
+· metrics_json  returns {"metrics": {}} when metrics.csv is missing or
+raises OSError on open (simulates file deleted mid-request)
+· metrics_stream yields a "done" chunk instead of crashing
+· log_content   returns None instead of crashing
+
+Rename / edit robustness
+· patch_run_meta raises ValueError for unknown keys and wrong value types
+· patch_run_meta returns False for an unknown run_id
+· patch_run_meta happy path persists changes to the sidecar on disk
+· do_PATCH HTTP handler returns 400 for bad fields / types, 404 for
+unknown runs, 500 when write_sidecar raises OSError, and 200 on success
+
+Serialization and data shapes
+· _serialize display_name resolution order
+· runs_json correct list structure and fields
+· metrics_json_bytes byte cache and NaN/Inf sanitization
+· _safe_dumps NaN/Inf → null
+· CSV edge cases (header-only, non-finite values)
+· log_content truncation behaviour
+· logs_index discovery and stream_id stability
+
+HTTP GET endpoints
+· All GET routes (/, /assets, /api/runs, /api/scan-status, /api/metrics,
+/api/logs, /api/log-content) return expected status codes
+· Path traversal blocked with 403
+· Missing parameters return 400
 """
 
 from __future__ import annotations
@@ -59,8 +58,10 @@ def _make_scanner(root: Path) -> RunScanner:
 
 
 def _inject_run(scanner: RunScanner, run_dir: Path, **sidecar_kw) -> str:
-    """Write a sidecar, create the run directory, and register the run in
-    the scanner's in-memory map without starting the background thread."""
+    """Write a sidecar and register the run in the scanner's in-memory map.
+
+    Does not start the background thread.
+    """
     from stable_pretraining.registry._sidecar import make_sidecar, write_sidecar
 
     run_dir.mkdir(parents=True, exist_ok=True)
@@ -218,7 +219,7 @@ def test_metrics_stream_yields_data_then_done(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_id = _inject_run(sc, tmp_path / "r1")
         (tmp_path / "r1" / "metrics.csv").write_text(
-            "step,val/acc\n" + "\n".join(f"{i},{i*0.01:.3f}" for i in range(20))
+            "step,val/acc\n" + "\n".join(f"{i},{i * 0.01:.3f}" for i in range(20))
         )
         chunks = [c for c in sc.metrics_stream(run_id) if c is not None]
         assert any(c.get("done") for c in chunks)
@@ -556,6 +557,8 @@ def _raw_get(base_url: str, path: str) -> int:
 
 
 class TestSafeDumps:
+    """Tests for the _safe_dumps NaN/Inf sanitization helper."""
+
     def test_nan_becomes_null(self):
         assert json.loads(_safe_dumps(float("nan"))) is None
 
@@ -589,6 +592,8 @@ def test_nested_dict_and_list(self):
 
 
 class TestSerialize:
+    """Tests for RunScanner._serialize display_name resolution and field shapes."""
+
     def test_explicit_display_name_wins(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "run1"
@@ -616,14 +621,14 @@ def test_explicit_display_name_wins(self, tmp_path):
     def test_falls_back_to_last_path_component(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "parent" / "child_run"
-        run_id = _inject_run(sc, run_dir)
+        _inject_run(sc, run_dir)
         runs = sc.runs_json()
         assert runs[0]["display_name"] == "child_run"
 
     def test_falls_back_to_full_run_id_when_no_slash(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "flat"
-        run_id = _inject_run(sc, run_dir)
+        _inject_run(sc, run_dir)
         runs = sc.runs_json()
         assert runs[0]["display_name"] == "flat"
 
@@ -671,6 +676,8 @@ def test_heartbeat_at_none_when_no_file(self, tmp_path):
 
 
 class TestRunsJson:
+    """Tests for RunScanner.runs_json list structure and field correctness."""
+
     def test_empty_when_no_runs(self, tmp_path):
         sc = _make_scanner(tmp_path)
         assert sc.runs_json() == []
@@ -713,6 +720,8 @@ def test_tags_and_notes_roundtrip(self, tmp_path):
 
 
 class TestMetricsCache:
+    """Tests for metrics_json mtime/size cache hit and invalidation."""
+
     def _write_csv(self, path: Path, rows: list[list]) -> None:
         import csv
 
@@ -748,7 +757,8 @@ def test_cache_invalidated_on_file_change(self, tmp_path):
             [["step", "loss"], ["1", "0.5"], ["2", "0.3"], ["3", "0.1"]],
         )
         # Touch so the mtime definitely changes (some filesystems have 1 s resolution).
-        import os, time
+        import os
+        import time
 
         os.utime(csv_path, (time.time() + 2, time.time() + 2))
 
@@ -760,6 +770,8 @@ def test_cache_invalidated_on_file_change(self, tmp_path):
 
 
 class TestMetricsJsonBytes:
+    """Tests for metrics_json_bytes byte cache and NaN/Inf sanitization."""
+
     def test_returns_valid_utf8_json(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "r"
@@ -798,6 +810,8 @@ def test_unknown_run_returns_none(self, tmp_path):
 
 
 class TestCsvEdgeCases:
+    """Tests for CSV parsing edge cases: empty files and non-finite float values."""
+
     def test_header_only_csv_returns_empty_metrics(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "r"
@@ -834,6 +848,8 @@ def test_nan_inf_sanitised_in_bytes(self, tmp_path):
 
 
 class TestLogsIndex:
+    """Tests for RunScanner.logs_index discovery and stream_id stability."""
+
     def test_unknown_run_returns_none(self, tmp_path):
         sc = _make_scanner(tmp_path)
         assert sc.logs_index("no-such") is None
@@ -879,6 +895,8 @@ def test_stream_id_is_stable_slug(self, tmp_path):
 
 
 class TestLogContentTruncation:
+    """Tests for RunScanner.log_content tail-truncation behaviour."""
+
     def test_small_file_returned_in_full(self, tmp_path):
         sc = _make_scanner(tmp_path)
         run_dir = tmp_path / "r"
@@ -925,6 +943,8 @@ def test_unknown_stream_returns_none(self, tmp_path):
 
 
 class TestGetEndpoints:
+    """Integration tests for all HTTP GET routes served by the web viewer."""
+
     def test_root_returns_200_html(self, web_server):
         _, base = web_server
         status, body, ctype = _do_get(base, "/")
diff --git a/stable_pretraining/web/scan.py b/stable_pretraining/web/scan.py
index f56686f38..abc1e878c 100644
--- a/stable_pretraining/web/scan.py
+++ b/stable_pretraining/web/scan.py
@@ -760,7 +760,9 @@ def patch_run_meta(self, run_id: str, patch: dict) -> bool:
         bad = set(patch) - self._MUTABLE_FIELDS
         if bad:
             raise ValueError(f"unknown patch fields: {sorted(bad)}")
-        if "display_name" in patch and not isinstance(patch["display_name"], (str, type(None))):
+        if "display_name" in patch and not isinstance(
+            patch["display_name"], (str, type(None))
+        ):
             raise ValueError("display_name must be a string")
         if "notes" in patch and not isinstance(patch["notes"], (str, type(None))):
             raise ValueError("notes must be a string")

From 155e9ef8a1b00a17ed6e918ab64829600cec5662 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 14:59:11 -0400
Subject: [PATCH 26/31] fixed final precommit errors from the jax
 implementation

---
 stable_pretraining/jax/backbone/resnet.py | 40 +++++++++++------------
 stable_pretraining/jax/backbone/small.py  |  6 ++--
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/stable_pretraining/jax/backbone/resnet.py b/stable_pretraining/jax/backbone/resnet.py
index 794332bde..244cb6b92 100644
--- a/stable_pretraining/jax/backbone/resnet.py
+++ b/stable_pretraining/jax/backbone/resnet.py
@@ -23,12 +23,12 @@
 from flax import nnx
 
 
-def _conv(din, dout, k, stride, rngs, padding, dtype=None):
+def _conv(din, d_out, k, stride, rngs, padding, dtype=None):
     # ``dtype`` is the *computation* dtype (e.g. bfloat16 for mixed precision);
     # ``param_dtype`` stays float32 so the weights are kept in full precision.
     return nnx.Conv(
         din,
-        dout,
+        d_out,
         kernel_size=(k, k),
         strides=(stride, stride),
         padding=padding,
@@ -46,12 +46,12 @@ def _bn(features, rngs, axis_name):
     )
 
 
-def _maybe_downsample(din, dout, stride, rngs, axis_name, dtype=None):
+def _maybe_downsample(din, d_out, stride, rngs, axis_name, dtype=None):
     """1x1-conv + BN projection shortcut, or ``None`` when shapes already match."""
-    if stride != 1 or din != dout:
+    if stride != 1 or din != d_out:
         return nnx.Sequential(
-            _conv(din, dout, 1, stride, rngs, "VALID", dtype),
-            _bn(dout, rngs, axis_name),
+            _conv(din, d_out, 1, stride, rngs, "VALID", dtype),
+            _bn(d_out, rngs, axis_name),
         )
     return None
 
@@ -61,15 +61,15 @@ class BasicBlock(nnx.Module):
 
     expansion = 1
 
-    def __init__(self, din, dout, stride, rngs, axis_name=None, dtype=None):
-        self.conv1 = _conv(din, dout, 3, stride, rngs, [(1, 1), (1, 1)], dtype)
-        self.bn1 = _bn(dout, rngs, axis_name)
-        self.conv2 = _conv(dout, dout, 3, 1, rngs, [(1, 1), (1, 1)], dtype)
-        self.bn2 = _bn(dout, rngs, axis_name)
+    def __init__(self, din, d_out, stride, rngs, axis_name=None, dtype=None):
+        self.conv1 = _conv(din, d_out, 3, stride, rngs, [(1, 1), (1, 1)], dtype)
+        self.bn1 = _bn(d_out, rngs, axis_name)
+        self.conv2 = _conv(d_out, d_out, 3, 1, rngs, [(1, 1), (1, 1)], dtype)
+        self.bn2 = _bn(d_out, rngs, axis_name)
         # Assign exactly once: mixing a static ``None`` then a module would
         # flip the attribute's pytree status and NNX rejects that.
         self.downsample = _maybe_downsample(
-            din, dout * self.expansion, stride, rngs, axis_name, dtype
+            din, d_out * self.expansion, stride, rngs, axis_name, dtype
         )
 
     def __call__(self, x):
@@ -84,15 +84,15 @@ class Bottleneck(nnx.Module):
 
     expansion = 4
 
-    def __init__(self, din, dout, stride, rngs, axis_name=None, dtype=None):
-        self.conv1 = _conv(din, dout, 1, 1, rngs, "VALID", dtype)
-        self.bn1 = _bn(dout, rngs, axis_name)
-        self.conv2 = _conv(dout, dout, 3, stride, rngs, [(1, 1), (1, 1)], dtype)
-        self.bn2 = _bn(dout, rngs, axis_name)
-        self.conv3 = _conv(dout, dout * self.expansion, 1, 1, rngs, "VALID", dtype)
-        self.bn3 = _bn(dout * self.expansion, rngs, axis_name)
+    def __init__(self, din, d_out, stride, rngs, axis_name=None, dtype=None):
+        self.conv1 = _conv(din, d_out, 1, 1, rngs, "VALID", dtype)
+        self.bn1 = _bn(d_out, rngs, axis_name)
+        self.conv2 = _conv(d_out, d_out, 3, stride, rngs, [(1, 1), (1, 1)], dtype)
+        self.bn2 = _bn(d_out, rngs, axis_name)
+        self.conv3 = _conv(d_out, d_out * self.expansion, 1, 1, rngs, "VALID", dtype)
+        self.bn3 = _bn(d_out * self.expansion, rngs, axis_name)
         self.downsample = _maybe_downsample(
-            din, dout * self.expansion, stride, rngs, axis_name, dtype
+            din, d_out * self.expansion, stride, rngs, axis_name, dtype
         )
 
     def __call__(self, x):
diff --git a/stable_pretraining/jax/backbone/small.py b/stable_pretraining/jax/backbone/small.py
index e6f5c076a..88a2ee5b8 100644
--- a/stable_pretraining/jax/backbone/small.py
+++ b/stable_pretraining/jax/backbone/small.py
@@ -10,18 +10,18 @@
 from flax import nnx
 
 
-def _conv_bn(din, dout, rngs, k=3, stride=1):
+def _conv_bn(din, d_out, rngs, k=3, stride=1):
     return nnx.Sequential(
         nnx.Conv(
             din,
-            dout,
+            d_out,
             kernel_size=(k, k),
             strides=(stride, stride),
             padding=[(k // 2, k // 2)] * 2,
             use_bias=False,
             rngs=rngs,
         ),
-        nnx.BatchNorm(dout, momentum=0.9, rngs=rngs),
+        nnx.BatchNorm(d_out, momentum=0.9, rngs=rngs),
     )
 
 

From 9daba1a15283f446f1ca272a63f9746daea01429 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Sun, 7 Jun 2026 15:30:58 -0400
Subject: [PATCH 27/31] small updates to pass all tests

---
 stable_pretraining/loggers/swanlab.py | 39 ++++++++++++++++-----------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/stable_pretraining/loggers/swanlab.py b/stable_pretraining/loggers/swanlab.py
index 00fb82e5d..e739eb469 100644
--- a/stable_pretraining/loggers/swanlab.py
+++ b/stable_pretraining/loggers/swanlab.py
@@ -64,22 +64,38 @@ class SwanLabLogger(_SwanLabLogger):
         them up automatically.
         """
 
+        # Upstream stores init kwargs in _init_kwargs; expose as _swanlab_init
+        # so our helpers and the checkpoint callback have a stable name.
+        @property
+        def _swanlab_init(self) -> Dict[str, Any]:
+            return self._init_kwargs  # type: ignore[attr-defined]
+
+        @property
+        def _project(self) -> Optional[str]:
+            return self._init_kwargs.get("project")  # type: ignore[attr-defined]
+
+        @property
+        def name(self) -> str:
+            return self._init_kwargs.get("project", "swanlab")  # type: ignore[attr-defined]
+
         @property
         def resume_info(self) -> Dict[str, Any]:
             """Snapshot of state needed to resume this run after requeue.
 
-            Prefers the live experiment's ``run_id`` when the experiment
-            has been initialised; falls back to whatever ``id=`` was passed
-            at construction.
+            Prefers the live experiment's ``id`` when the experiment has been
+            initialised; falls back to whatever ``id=`` was passed at
+            construction.
             """
             live_id = None
             exp = getattr(self, "_experiment", None)
             if exp is not None:
                 public = getattr(exp, "public", None)
                 live_id = getattr(public, "run_id", None) if public else None
-            init_cfg = getattr(self, "_swanlab_init", {}) or {}
+                if live_id is None:
+                    live_id = getattr(exp, "id", None)
+            init_cfg = self._swanlab_init
             return {
-                "project": getattr(self, "_project", init_cfg.get("project")),
+                "project": self._project,
                 "experiment_name": init_cfg.get("experiment_name"),
                 "group": init_cfg.get("group"),
                 "id": live_id or init_cfg.get("id"),
@@ -88,20 +104,13 @@ def resume_info(self) -> Dict[str, Any]:
         def set_resume(self, id: str) -> None:
             """Configure this logger to resume a previous experiment.
 
-            Mutates the underlying ``_swanlab_init`` dict so that the next
+            Mutates the underlying ``_init_kwargs`` dict so that the next
             call to ``swanlab.init()`` (triggered on first access to
             ``self.experiment``) picks up ``id=<id>`` with
             ``resume="must"``.  Call this *before* any logging happens.
             """
-            init_cfg = getattr(self, "_swanlab_init", None)
-            if init_cfg is None:
-                # Upstream shape changed — store on the instance so that
-                # users relying on set_resume() still see a well-defined
-                # attribute for debugging.
-                self._swanlab_init = {"id": id, "resume": "must"}
-                return
-            init_cfg["id"] = id
-            init_cfg["resume"] = "must"
+            self._init_kwargs["id"] = id  # type: ignore[attr-defined]
+            self._init_kwargs["resume"] = "must"  # type: ignore[attr-defined]
 
 else:
     # Placeholder that raises on instantiation so imports don't break when

From 29c0474467d20bfc0773a7d53995fbbf89d47257 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Mon, 8 Jun 2026 14:57:18 -0400
Subject: [PATCH 28/31] fixing failing integration tests

---
 stable_pretraining/data/transforms.py | 54 +++++++++++++--------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/stable_pretraining/data/transforms.py b/stable_pretraining/data/transforms.py
index d565facfd..c7f5fa841 100644
--- a/stable_pretraining/data/transforms.py
+++ b/stable_pretraining/data/transforms.py
@@ -351,9 +351,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros(4)
             return x
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(
             [params["i"], params["j"], params["h"], params["w"]]
@@ -383,9 +383,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(
             [
@@ -418,9 +418,9 @@ def __call__(self, x):
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros(8)
             return x
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         coeffs = params["coefficients"]
         x[self.get_name(x)] = torch.Tensor(coeffs)
@@ -442,9 +442,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.Tensor([0.0, 0.0])
             return x
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor([self.mean, self.sigma])
         return x
@@ -472,9 +472,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros((2,))
             return x
-        params = self.make_params([])
+        params = self._get_params([])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(params["sigma"])
         return x
@@ -612,7 +612,7 @@ def __init__(
 
     def __call__(self, x):
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), []), self.target
+            x, self._transform(self.nested_get(x, self.source), []), self.target
         )
         return x
 
@@ -640,9 +640,9 @@ def __call__(self, x) -> Any:
             self.nested_set(x, self.nested_get(x, self.source), self.target)
             x[self.get_name(x)] = torch.zeros(8)
             return x
-        params = self.make_params([])
+        params = self._get_params([])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         brightness_factor = params["brightness_factor"]
         contrast_factor = params["contrast_factor"]
@@ -673,9 +673,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        angle = self.make_params([])
+        angle = self._get_params([])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), angle), self.target
+            x, self._transform(self.nested_get(x, self.source), angle), self.target
         )
         x[self.get_name(x)] = angle
         return x
@@ -719,9 +719,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), params), self.target
+            x, self._transform(self.nested_get(x, self.source), params), self.target
         )
         values = []
         values.append(params["needs_crop"])
@@ -778,15 +778,15 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self.make_params([self.nested_get(x, self.source)])
+        params = self._get_params([self.nested_get(x, self.source)])
 
         candidates = self.nested_get(x, self.source)
         if type(candidates) in [tuple, list]:
-            out = [self.transform(c, params) for c in candidates]
+            out = [self._transform(c, params) for c in candidates]
             self.nested_set(x, out, self.target)
         else:
             self.nested_set(
-                x, self.transform(self.nested_get(x, self.source), params), self.target
+                x, self._transform(self.nested_get(x, self.source), params), self.target
             )
         values = []
         values.append(params["top"])
@@ -906,7 +906,7 @@ def __init__(self, size, source: str = "image", target: str = "image"):
 
     def __call__(self, x):
         self.nested_set(
-            x, self.transform(self.nested_get(x, self.source), []), self.target
+            x, self._transform(self.nested_get(x, self.source), []), self.target
         )
         return x
 
@@ -940,12 +940,12 @@ def __init__(
     ):
         super().__init__()
         self.seed_offset = seed_offset
-        self._transform = transform
+        self.transform = transform
         self.key = key
 
     def __call__(self, x):
         with random_seed(x["idx"] + self.seed_offset):
-            x = self._transform(x)
+            x = self.transform(x)
         return x
 
 
@@ -954,18 +954,18 @@ class Conditional(Transform):
 
     def __init__(self, transform, condition_key, apply_on_true=True):
         super().__init__()
-        self._transform = transform
+        self.transform = transform
         self.condition_key = condition_key
         self.apply_on_true = apply_on_true
 
     def __call__(self, x):
         if x[self.condition_key] and self.apply_on_true:
-            return self._transform(x)
+            return self.transform(x)
         elif not x[self.condition_key] and not self.apply_on_true:
-            return self._transform(x)
+            return self.transform(x)
         # if the transform is not applied we still inform the user
         # otherwise collate_fn will complain
-        x[self._transform.get_name(x)] = self._transform.BYPASS_VALUE
+        x[self.transform.get_name(x)] = self.transform.BYPASS_VALUE
         return x
 
 

From d271b05a2f3b9cf0b1debf002e6792b1975c3c04 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Mon, 8 Jun 2026 15:03:31 -0400
Subject: [PATCH 29/31] Revert "fixing failing integration tests"

This reverts commit 29c0474467d20bfc0773a7d53995fbbf89d47257.
---
 stable_pretraining/data/transforms.py | 54 +++++++++++++--------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/stable_pretraining/data/transforms.py b/stable_pretraining/data/transforms.py
index c7f5fa841..d565facfd 100644
--- a/stable_pretraining/data/transforms.py
+++ b/stable_pretraining/data/transforms.py
@@ -351,9 +351,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros(4)
             return x
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(
             [params["i"], params["j"], params["h"], params["w"]]
@@ -383,9 +383,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(
             [
@@ -418,9 +418,9 @@ def __call__(self, x):
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros(8)
             return x
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         coeffs = params["coefficients"]
         x[self.get_name(x)] = torch.Tensor(coeffs)
@@ -442,9 +442,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.Tensor([0.0, 0.0])
             return x
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor([self.mean, self.sigma])
         return x
@@ -472,9 +472,9 @@ def __call__(self, x) -> Any:
         if self.p < 1 and torch.rand(1) >= self.p:
             x[self.get_name(x)] = torch.zeros((2,))
             return x
-        params = self._get_params([])
+        params = self.make_params([])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         x[self.get_name(x)] = torch.Tensor(params["sigma"])
         return x
@@ -612,7 +612,7 @@ def __init__(
 
     def __call__(self, x):
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), []), self.target
+            x, self.transform(self.nested_get(x, self.source), []), self.target
         )
         return x
 
@@ -640,9 +640,9 @@ def __call__(self, x) -> Any:
             self.nested_set(x, self.nested_get(x, self.source), self.target)
             x[self.get_name(x)] = torch.zeros(8)
             return x
-        params = self._get_params([])
+        params = self.make_params([])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         brightness_factor = params["brightness_factor"]
         contrast_factor = params["contrast_factor"]
@@ -673,9 +673,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        angle = self._get_params([])
+        angle = self.make_params([])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), angle), self.target
+            x, self.transform(self.nested_get(x, self.source), angle), self.target
         )
         x[self.get_name(x)] = angle
         return x
@@ -719,9 +719,9 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), params), self.target
+            x, self.transform(self.nested_get(x, self.source), params), self.target
         )
         values = []
         values.append(params["needs_crop"])
@@ -778,15 +778,15 @@ def __init__(
         self.target = target
 
     def __call__(self, x):
-        params = self._get_params([self.nested_get(x, self.source)])
+        params = self.make_params([self.nested_get(x, self.source)])
 
         candidates = self.nested_get(x, self.source)
         if type(candidates) in [tuple, list]:
-            out = [self._transform(c, params) for c in candidates]
+            out = [self.transform(c, params) for c in candidates]
             self.nested_set(x, out, self.target)
         else:
             self.nested_set(
-                x, self._transform(self.nested_get(x, self.source), params), self.target
+                x, self.transform(self.nested_get(x, self.source), params), self.target
             )
         values = []
         values.append(params["top"])
@@ -906,7 +906,7 @@ def __init__(self, size, source: str = "image", target: str = "image"):
 
     def __call__(self, x):
         self.nested_set(
-            x, self._transform(self.nested_get(x, self.source), []), self.target
+            x, self.transform(self.nested_get(x, self.source), []), self.target
         )
         return x
 
@@ -940,12 +940,12 @@ def __init__(
     ):
         super().__init__()
         self.seed_offset = seed_offset
-        self.transform = transform
+        self._transform = transform
         self.key = key
 
     def __call__(self, x):
         with random_seed(x["idx"] + self.seed_offset):
-            x = self.transform(x)
+            x = self._transform(x)
         return x
 
 
@@ -954,18 +954,18 @@ class Conditional(Transform):
 
     def __init__(self, transform, condition_key, apply_on_true=True):
         super().__init__()
-        self.transform = transform
+        self._transform = transform
         self.condition_key = condition_key
         self.apply_on_true = apply_on_true
 
     def __call__(self, x):
         if x[self.condition_key] and self.apply_on_true:
-            return self.transform(x)
+            return self._transform(x)
         elif not x[self.condition_key] and not self.apply_on_true:
-            return self.transform(x)
+            return self._transform(x)
         # if the transform is not applied we still inform the user
         # otherwise collate_fn will complain
-        x[self.transform.get_name(x)] = self.transform.BYPASS_VALUE
+        x[self._transform.get_name(x)] = self._transform.BYPASS_VALUE
         return x
 
 

From 1020dd73e5c6aa641e5b9907330ffa23f28ac6d3 Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 18 Jun 2026 18:25:15 -0400
Subject: [PATCH 30/31] first round of additional tests and fixes

---
 stable_pretraining/tests/unit/test_web.py | 157 +++++++++++++++++++++-
 1 file changed, 155 insertions(+), 2 deletions(-)

diff --git a/stable_pretraining/tests/unit/test_web.py b/stable_pretraining/tests/unit/test_web.py
index 39cfbbdf8..450c2fa5a 100644
--- a/stable_pretraining/tests/unit/test_web.py
+++ b/stable_pretraining/tests/unit/test_web.py
@@ -423,6 +423,22 @@ def test_patch_persists_tags_to_sidecar(self, tmp_path):
         on_disk = read_sidecar(sidecar_path(tmp_path / "r1"))
         assert on_disk["tags"] == ["a", "b"]
 
+    def test_patch_persists_notes_to_sidecar(self, tmp_path):
+        from stable_pretraining.registry._sidecar import read_sidecar, sidecar_path
+
+        sc, run_id = self._setup(tmp_path)
+        sc.patch_run_meta(run_id, {"notes": "my experiment notes"})
+        on_disk = read_sidecar(sidecar_path(tmp_path / "r1"))
+        assert on_disk["notes"] == "my experiment notes"
+
+    def test_patch_persists_archived_to_sidecar(self, tmp_path):
+        from stable_pretraining.registry._sidecar import read_sidecar, sidecar_path
+
+        sc, run_id = self._setup(tmp_path)
+        sc.patch_run_meta(run_id, {"archived": True})
+        on_disk = read_sidecar(sidecar_path(tmp_path / "r1"))
+        assert on_disk["archived"] is True
+
     def test_patch_updates_in_memory_sidecar(self, tmp_path):
         """In-memory run.sidecar must reflect the patch immediately."""
         sc, run_id = self._setup(tmp_path)
@@ -514,6 +530,59 @@ def test_invalid_json_body_returns_400(self, web_server):
             status = exc.code
         assert status == 400
 
+    def test_body_too_large_returns_400(self, web_server):
+        """Content-Length > 64 KiB must be rejected before reading the body."""
+        _, base = web_server
+        port = int(base.rsplit(":", 1)[-1])
+        big_len = 64 * 1024 + 1
+        conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
+        conn.putrequest("PATCH", "/api/run-meta")
+        conn.putheader("Content-Type", "application/json")
+        conn.putheader("Content-Length", str(big_len))
+        conn.endheaders()
+        # Don't send the body — the server rejects on the header alone and
+        # returns 400 without calling rfile.read(), so no body is needed.
+        resp = conn.getresponse()
+        status = resp.status
+        resp.read()
+        conn.close()
+        assert status == 400
+
+    def test_non_dict_json_body_returns_400(self, web_server):
+        """A valid JSON array or scalar body (not an object) must return 400."""
+        _, base = web_server
+        body = json.dumps([1, 2, 3]).encode()
+        req = urllib.request.Request(
+            base + "/api/run-meta",
+            data=body,
+            method="PATCH",
+            headers={
+                "Content-Type": "application/json",
+                "Content-Length": str(len(body)),
+            },
+        )
+        try:
+            with urllib.request.urlopen(req) as resp:
+                status = resp.status
+        except urllib.error.HTTPError as exc:
+            status = exc.code
+        assert status == 400
+
+    def test_run_id_non_string_returns_400(self, tmp_path, web_server):
+        """run_id present but not a string must return 400."""
+        scanner, base = web_server
+        _inject_run(scanner, tmp_path / "r7")
+        status, body = _do_patch(base, {"run_id": 123, "display_name": "x"})
+        assert status == 400
+        assert "error" in body
+
+    def test_run_id_empty_string_returns_400(self, web_server):
+        """run_id present but empty string must return 400."""
+        _, base = web_server
+        status, body = _do_patch(base, {"run_id": "", "display_name": "x"})
+        assert status == 400
+        assert "error" in body
+
     def test_wrong_path_returns_404(self, web_server):
         _, base = web_server
         req = urllib.request.Request(
@@ -529,6 +598,37 @@ def test_wrong_path_returns_404(self, web_server):
             assert exc.code == 404
 
 
+# ── TestPatchSSEPublish ───────────────────────────────────────────────────────
+
+
+class TestPatchSSEPublish:
+    """patch_run_meta must fire an SSE update on success and stay silent on failure."""
+
+    def test_publishes_update_on_success(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        run_id = _inject_run(sc, tmp_path / "r1")
+        q = sc.subscribe()
+        try:
+            sc.patch_run_meta(run_id, {"display_name": "new-name"})
+        finally:
+            sc.unsubscribe(q)
+        assert not q.empty()
+        event = q.get_nowait()
+        assert event["type"] == "update"
+        assert run_id in event["data"]["changed"]
+        assert event["data"]["removed"] == []
+
+    def test_does_not_publish_on_unknown_run(self, tmp_path):
+        sc = _make_scanner(tmp_path)
+        q = sc.subscribe()
+        try:
+            result = sc.patch_run_meta("no-such-run", {"display_name": "x"})
+        finally:
+            sc.unsubscribe(q)
+        assert result is False
+        assert q.empty()
+
+
 # ── GET helpers ───────────────────────────────────────────────────────────────
 
 
@@ -671,6 +771,57 @@ def test_heartbeat_at_none_when_no_file(self, tmp_path):
         runs = sc.runs_json()
         assert runs[0]["heartbeat_at"] is None
 
+    def test_ended_at_roundtrip(self, tmp_path):
+        from stable_pretraining.registry._sidecar import make_sidecar, write_sidecar
+
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run_ended"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        run_id = str(run_dir.relative_to(sc.root))
+        ended_ts = 1_700_000_000.0
+        sid = make_sidecar(run_id=run_id, run_dir=str(run_dir), ended_at=ended_ts)
+        write_sidecar(run_dir, sid)
+        run = _Run(
+            run_id=run_id,
+            run_dir=run_dir,
+            sidecar_mtime=0.0,
+            metrics_mtime=0.0,
+            metrics_size=0,
+            sidecar=sid,
+        )
+        with sc._lock:
+            sc._runs[run_id] = run
+        runs = sc.runs_json()
+        assert runs[0]["ended_at"] == pytest.approx(ended_ts)
+
+    def test_heartbeat_at_roundtrip(self, tmp_path):
+        from stable_pretraining.registry._sidecar import (
+            make_sidecar,
+            touch_heartbeat,
+            write_sidecar,
+        )
+
+        sc = _make_scanner(tmp_path)
+        run_dir = tmp_path / "run_hb"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        run_id = str(run_dir.relative_to(sc.root))
+        sid = make_sidecar(run_id=run_id, run_dir=str(run_dir))
+        write_sidecar(run_dir, sid)
+        touch_heartbeat(run_dir)
+        hb_mtime = (run_dir / "heartbeat").stat().st_mtime
+        run = _Run(
+            run_id=run_id,
+            run_dir=run_dir,
+            sidecar_mtime=0.0,
+            metrics_mtime=0.0,
+            metrics_size=0,
+            sidecar=sid,
+        )
+        with sc._lock:
+            sc._runs[run_id] = run
+        runs = sc.runs_json()
+        assert runs[0]["heartbeat_at"] == pytest.approx(hb_mtime)
+
 
 # ── TestRunsJson ──────────────────────────────────────────────────────────────
 
@@ -736,8 +887,10 @@ def test_cache_hit_avoids_reparse(self, tmp_path):
         self._write_csv(csv_path, [["step", "loss"], ["1", "0.5"], ["2", "0.3"]])
 
         first = sc.metrics_json(run_id)
-        # Second call should hit cache; patch open to detect if file is re-read.
-        with patch("builtins.open", side_effect=AssertionError("file re-opened")):
+        # Second call must hit the mtime/size cache without re-opening the file.
+        # Patch Path.open (the actual call site) — builtins.open is not used by
+        # mpath.open() so patching it would leave the cache test with no teeth.
+        with patch.object(Path, "open", side_effect=AssertionError("file re-opened")):
             second = sc.metrics_json(run_id)
         assert first == second
 

From be2201c2a299289df9ce69b53f44506db439a68e Mon Sep 17 00:00:00 2001
From: MarcelMatsal <marcel_mateos_salles@brown.edu>
Date: Thu, 18 Jun 2026 18:36:39 -0400
Subject: [PATCH 31/31] new safety fix to frontend

---
 stable_pretraining/web/assets/app.js | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/stable_pretraining/web/assets/app.js b/stable_pretraining/web/assets/app.js
index c58f3b911..7b2c743f2 100644
--- a/stable_pretraining/web/assets/app.js
+++ b/stable_pretraining/web/assets/app.js
@@ -3404,6 +3404,18 @@
     const metricNames = visibleMetricNames();
     if (!visIds.length || !metricNames.length) return;
 
+    // Proper RFC-4180 quoting for a single cell value.
+    // - Quotes when the value contains , / " / newline / CR.
+    // - Escapes inner " by doubling.
+    // - Prefixes values that start with = + - @ with a tab so spreadsheet
+    //   apps (Excel, Google Sheets) don't interpret them as formulas.
+    function csvCell(v) {
+      let s = v == null ? '' : String(v);
+      if (s.length > 0 && '=+-@'.includes(s[0])) s = '\t' + s;
+      if (/[,"\n\r]/.test(s)) return '"' + s.replace(/"/g, '""') + '"';
+      return s;
+    }
+
     const rows = ['run_id,run_name,metric,step,epoch,value'];
     for (const id of visIds) {
       const run = state.runs.get(id);
@@ -3417,9 +3429,9 @@
           const step  = col.step[i]  ?? '';
           const epoch = col.epoch[i] ?? '';
           const val   = col.y[i]     ?? '';
-          const escapedName   = name.includes(',')   ? `"${name.replace(/"/g, '""')}"` : name;
-          const escapedMetric = metric.includes(',') ? `"${metric.replace(/"/g, '""')}"` : metric;
-          rows.push(`${id},${escapedName},${escapedMetric},${step},${epoch},${val}`);
+          // step / epoch / val are always numbers or '' from the CSV parser,
+          // so they need no quoting or injection guard.
+          rows.push([csvCell(id), csvCell(name), csvCell(metric), step, epoch, val].join(','));
         }
       }
     }