mm/dr-sandbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: mm:fe495656514196fca6fc4e9ac013093b30dd24c3
Branches Tags
mm:main
..
compare: mm:f524dcce51fcdc51e01410e8cfb893db5f3eb043
Branches Tags
mm:main

No commits in common. "fe495656514196fca6fc4e9ac013093b30dd24c3" and "f524dcce51fcdc51e01410e8cfb893db5f3eb043" have entirely different histories.
fe49565651 ... f524dcce51

9 changed files with 12 additions and 1799 deletions
Show Stats Expand all files Collapse all files

237
app/web/main.py
View File

@ -10,20 +10,16 @@ framework; hand-written styles.
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import re
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from app.web.plotly_parse import parse_plotly_run
import httpx
from fastapi import FastAPI, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, Response
from fastapi import FastAPI, Form, Request
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from sklearn.datasets import (
@ -214,67 +210,6 @@ REDUCERS: Dict[str, Dict[str, Any]] = {
("rotation", "str", "", ["", "varimax", "quartimax"], "Empty = None."),
],
},
"sklearn.decomposition.KernelPCA": {
"pkg": "sklearn",
"label": "KernelPCA",
"blurb": "Non-linear PCA via the kernel trick. Deterministic; kernel choice shapes the output.",
"key": [
("n_components", "int", 2, None, "Locked."),
("kernel", "str", "rbf", ["linear", "poly", "rbf", "sigmoid", "cosine"], None),
("random_state", "int", 42, None, None),
],
"advanced": [
("gamma", "str_or_float", "", None, "Empty = 1/n_features."),
("degree", "int", 3, None, None),
("coef0", "float", 1.0, None, None),
("alpha", "float", 1.0, None, None),
],
},
"sklearn.manifold.Isomap": {
"pkg": "sklearn",
"label": "Isomap",
"blurb": "Geodesic-distance manifold learning via shortest paths on a k-NN graph.",
"key": [
("n_components", "int", 2, None, "Locked."),
("n_neighbors", "int", 5, None, None),
],
"advanced": [
("metric", "str", "minkowski", None, None),
("p", "int", 2, None, "Minkowski power (1 = manhattan, 2 = euclidean)."),
("path_method", "str", "auto", ["auto", "FW", "D"], "Floyd-Warshall / Dijkstra / auto."),
("neighbors_algorithm", "str", "auto", ["auto", "ball_tree", "kd_tree", "brute"], None),
],
},
"sklearn.manifold.MDS": {
"pkg": "sklearn",
"label": "MDS",
"blurb": "Multidimensional scaling. Preserves pairwise distances; O(n²) memory.",
"key": [
("n_components", "int", 2, None, "Locked."),
("n_init", "int", 4, None, None),
("random_state", "int", 42, None, None),
],
"advanced": [
("max_iter", "int", 300, None, None),
("metric_mds", "bool", True, None, "Metric (True) vs non-metric MDS."),
("metric", "str", "euclidean", None, None),
("eps", "float", 1e-6, None, "Convergence tolerance."),
],
},
"sklearn.manifold.SpectralEmbedding": {
"pkg": "sklearn",
"label": "SpectralEmbedding",
"blurb": "Laplacian eigenmaps on an affinity graph. What UMAP uses for initialisation.",
"key": [
("n_components", "int", 2, None, "Locked."),
("affinity", "str", "nearest_neighbors", ["nearest_neighbors", "rbf"], None),
("random_state", "int", 42, None, None),
],
"advanced": [
("n_neighbors", "int_or_null", "", None, "For affinity=nearest_neighbors. Empty = n/10."),
("gamma", "str_or_float", "", None, "For affinity=rbf. Empty = 1/n_features."),
],
},
"sklearn.manifold.TSNE": {
"pkg": "sklearn",
"label": "t-SNE",
@ -360,16 +295,6 @@ REDUCERS: Dict[str, Dict[str, Any]] = {
("weight_adj", "float", 500.0, None, None),
],
},
"sklearn.random_projection.GaussianRandomProjection": {
"pkg": "sklearn",
"label": "GaussianRandomProjection",
"blurb": "Johnson-Lindenstrauss baseline. Cheap, distance-preserving in expectation, structure-agnostic.",
"key": [
("n_components", "int", 2, None, "Locked."),
("random_state", "int", 42, None, None),
],
"advanced": [],
},
}
@ -450,13 +375,6 @@ def build_embed_args(reducer_key: str, form: Dict[str, str]) -> Dict[str, Any]:
# ---------------------------------------------------------------------------
def embed_args_hash(embed_args: Optional[Dict[str, Any]]) -> str:
"""8-hex digest of an embed_args dict (keys sorted). Stems incorporate
this so runs that differ only in embed_args get distinct output files."""
s = json.dumps(embed_args or {}, sort_keys=True, default=str)
return hashlib.sha1(s.encode()).hexdigest()[:8]
def synthesize_output_paths(
generator_path: str,
embedder: str,
@ -464,34 +382,14 @@ def synthesize_output_paths(
num_timesteps: int,
jitter_scale: float,
seed: int,
embed_args: Optional[Dict[str, Any]] = None,
) -> Tuple[str, str]:
gen = generator_path.split(".")[-1]
emb = embedder.split(".")[-1]
ref = f"{gen}_Reference_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}.html"
base = f"{gen}_{emb}_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}"
if embed_args is None:
embf = f"{base}.html"
else:
embf = f"{base}_{embed_args_hash(embed_args)}.html"
embf = f"{gen}_{emb}_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}.html"
return ref, embf
def _resolve_emb_file(synthesized: str) -> str:
"""Disk-backed fallback: prefer the synthesized (hashed) name; if that
doesn't exist on disk but an older hash-less variant does, return that
so pre-hash runs still render in the UI."""
if (FIGS_DIR / synthesized).exists():
return synthesized
# Strip trailing _<8hex>.html to get the legacy name.
m = re.match(r"^(?P<base>.+)_[0-9a-f]{8}\.html$", synthesized)
if m:
legacy = m.group("base") + ".html"
if (FIGS_DIR / legacy).exists():
return legacy
return synthesized # new / still-running run; let emb_exists resolve
# ---------------------------------------------------------------------------
# Prefect client
# ---------------------------------------------------------------------------
@ -614,10 +512,7 @@ def _run_view(run: Dict[str, Any]) -> Dict[str, Any]:
int(params.get("num_timesteps", params.get("num_snapshots", 48))),
float(params.get("jitter_scale", 0.01)),
int(params.get("seed", 42)),
embed_args=params.get("embed_args") or {},
)
# Older runs may lack the hash suffix; prefer legacy name on disk.
emb_file = _resolve_emb_file(emb_file)
except Exception:
ref_file, emb_file = None, None
@ -642,22 +537,6 @@ def _run_view(run: Dict[str, Any]) -> Dict[str, Any]:
}
def _mark_stale_views(views: List[Dict[str, Any]]) -> None:
"""Flag runs whose emb HTML was overwritten by a newer sibling with
identical params. Prefect returns runs sorted by START_TIME_DESC, so the
first occurrence of each stem is authoritative; later ones are stale.
Mutates views in place."""
seen: set = set()
for v in views:
stem = v["emb_file"][:-5] if v.get("emb_file") else None
if stem and stem in seen:
v["stale"] = True
else:
v["stale"] = False
if stem:
seen.add(stem)
def _reducer_choices() -> List[Dict[str, str]]:
return [
{"key": k, "label": spec["label"], "blurb": spec["blurb"]}
@ -679,7 +558,6 @@ async def index(request: Request) -> HTMLResponse:
runs = await PREFECT.recent_runs(client, limit=10)
dep_id = await PREFECT.deployment_id(client)
views = [_run_view(r) for r in runs]
_mark_stale_views(views)
return templates.TemplateResponse(
request,
"index.html",
@ -716,7 +594,6 @@ async def runs_partial(request: Request) -> HTMLResponse:
async with httpx.AsyncClient(timeout=5.0) as client:
runs = await PREFECT.recent_runs(client, limit=10)
views = [_run_view(r) for r in runs]
_mark_stale_views(views)
return templates.TemplateResponse(
request, "_runs.html", {"runs": views}
)
@ -782,27 +659,6 @@ async def submit(request: Request) -> HTMLResponse:
embed_args = build_embed_args(reducer, data)
# Reject submissions whose output path would overwrite an existing fig.
# The stem now includes an 8-hex hash of embed_args, so UMAP(n_neighbors=5)
# and UMAP(n_neighbors=15) produce distinct files. Check both the hashed
# path (new runs) and the legacy hashless path (pre-hash runs) so users
# can't accidentally duplicate against a pre-hash fig either.
_, hashed_emb = synthesize_output_paths(
generator_path, reducer, num_points, num_timesteps, jitter_scale, seed,
embed_args=embed_args,
)
_, legacy_emb = synthesize_output_paths(
generator_path, reducer, num_points, num_timesteps, jitter_scale, seed,
)
for candidate in (hashed_emb, legacy_emb):
if (FIGS_DIR / candidate).exists():
return HTMLResponse(
f"<div class='flash err'>a run with matching params already "
f"exists (<code>{candidate}</code>). change a param or delete "
f"the fig first.</div>",
status_code=409,
)
parameters: Dict[str, Any] = {
"num_points": num_points,
"num_timesteps": num_timesteps,
@ -832,8 +688,7 @@ async def submit(request: Request) -> HTMLResponse:
)
ref_file, emb_file = synthesize_output_paths(
generator_path, reducer, num_points, num_timesteps, jitter_scale, seed,
embed_args=embed_args,
generator_path, reducer, num_points, num_timesteps, jitter_scale, seed
)
RUN_OUTPUTS[run["id"]] = {"ref": ref_file, "embed": emb_file}
@ -841,7 +696,6 @@ async def submit(request: Request) -> HTMLResponse:
async with httpx.AsyncClient(timeout=5.0) as client:
runs = await PREFECT.recent_runs(client, limit=10)
views = [_run_view(r) for r in runs]
_mark_stale_views(views)
return templates.TemplateResponse(
request,
"_runs.html",
@ -879,89 +733,6 @@ async def metrics_json() -> JSONResponse:
return JSONResponse(_scan_metrics())
_STEM_RE = re.compile(
r"^make_[A-Za-z_]+?_[A-Za-z]+_N\d+_T\d+_J[\d.]+_s\d+(?:_[0-9a-f]{8})?$"
)
# Map short generator name ("make_blobs") to its DATASET_META entry.
# swiss_roll and swiss_roll_hole collide on path; first wins (plain variant).
_GEN_TO_META: Dict[str, Dict[str, Any]] = {}
for _m in DATASET_META.values():
_GEN_TO_META.setdefault(_m["path"].rsplit(".", 1)[-1], _m)
def _enrich_with_labels(d: Dict[str, Any]) -> Dict[str, Any]:
"""Attach per-point class/continuous labels by regenerating the dataset
with the same (generator, n_samples, kwargs). The stem's `seed` drives
jitter — NOT generator — so we always use random_state=0 to match the
flow's _DEFAULT_GENERATOR_KWARGS. Jitter-added points (id >= num_points)
get None so the client renders them as black."""
meta = _GEN_TO_META.get(d["meta"].get("generator") or "")
if not meta:
return d
try:
mod_path, cls_name = meta["path"].rsplit(".", 1)
fn = getattr(importlib.import_module(mod_path), cls_name)
N = int(d["meta"]["num_points"])
_, gen_labels = fn(n_samples=N, random_state=0, **meta["kwargs"])
out_labels: List[Optional[float]] = []
for pid in d["point_ids"]:
if isinstance(pid, int) and 0 <= pid < N:
v = gen_labels[pid]
out_labels.append(float(v) if hasattr(v, "item") or isinstance(v, (int, float)) else None)
else:
out_labels.append(None)
d["labels"] = out_labels
d["label_kind"] = meta["kind"]
except Exception:
pass
return d
@lru_cache(maxsize=32)
def _cached_frames(stem: str) -> str:
"""Return the frames dict as a JSON string. Prefers a <stem>.frames.json
sidecar (emitted by new flow runs); falls back to parsing <stem>.html
(for pre-sidecar runs). Either way, enriches with dataset labels."""
sidecar = FIGS_DIR / f"{stem}.frames.json"
if sidecar.is_file():
d = json.loads(sidecar.read_text(encoding="utf-8"))
else:
html = FIGS_DIR / f"{stem}.html"
d = parse_plotly_run(html)
d = _enrich_with_labels(d)
return json.dumps(d, separators=(",", ":"))
@app.get("/api/runs/{stem}/frames.json")
async def run_frames(stem: str) -> Response:
if not _STEM_RE.match(stem):
raise HTTPException(400, f"malformed stem: {stem!r}")
if not (FIGS_DIR / f"{stem}.frames.json").is_file() and not (FIGS_DIR / f"{stem}.html").is_file():
raise HTTPException(404, f"no such run: {stem}")
try:
payload = _cached_frames(stem)
except Exception as e:
raise HTTPException(500, f"parse failed: {e}")
return Response(
content=payload,
media_type="application/json",
headers={"Cache-Control": "no-cache"},
)
@app.get("/compare", response_class=HTMLResponse)
async def compare_page(request: Request, a: str = "", b: str = "") -> HTMLResponse:
for label, stem in (("a", a), ("b", b)):
if not stem or not _STEM_RE.match(stem):
raise HTTPException(400, f"missing or malformed stem for {label!r}")
if not (FIGS_DIR / f"{stem}.html").is_file():
raise HTTPException(404, f"no such run: {stem}")
return templates.TemplateResponse(
request, "compare.html", {"stem_a": a, "stem_b": b}
)
@app.get("/health")
async def health() -> JSONResponse:
async with httpx.AsyncClient(timeout=3.0) as client:

290
app/web/plotly_parse.py
View File

@ -1,290 +0,0 @@
"""Parse flow-emitted Plotly HTML files into a compact per-frame JSON dict.
The embedding-sandbox flow writes one standalone HTML per run at
``figs/<stem>.html``. Each HTML contains a ``Plotly.newPlot(...)`` call (which
holds the initial frame) and a ``Plotly.addFrames(...)`` call (which holds the
full animation — including a redundant copy of frame 0). Point data is encoded
with plotly's base64+dtype ("bdata") scheme, so we decode it with ``struct``.
The module is stdlib-only.
"""
from __future__ import annotations
import base64
import json
import re
import struct
from pathlib import Path
_STEM_RE = re.compile(
r"^(?P<gen>make_.+?)_(?P<emb>[A-Za-z]+)_N(?P<n>\d+)_T(?P<t>\d+)"
r"_J(?P<j>[\d.]+)_s(?P<s>\d+)(?:_(?P<h>[0-9a-f]{8}))?$"
)
# plotly's typed-array dtype -> (struct format char, item size bytes)
_DTYPE_MAP = {
"i1": ("b", 1),
"u1": ("B", 1),
"i2": ("h", 2),
"u2": ("H", 2),
"i4": ("i", 4),
"u4": ("I", 4),
"i8": ("q", 8),
"u8": ("Q", 8),
"f4": ("f", 4),
"f8": ("d", 8),
}
def _decode_bdata(obj):
# Accepts either a plotly typed-array dict {"dtype","bdata",...} or a plain list.
if isinstance(obj, list):
return list(obj)
if not isinstance(obj, dict) or "bdata" not in obj:
raise ValueError(f"expected bdata object or list, got {type(obj).__name__}")
dt = obj.get("dtype")
if dt not in _DTYPE_MAP:
raise ValueError(f"unsupported dtype {dt!r}")
fmt, size = _DTYPE_MAP[dt]
raw = base64.b64decode(obj["bdata"])
count = len(raw) // size
return list(struct.unpack("<" + fmt * count, raw[: count * size]))
def _extract_call_args(txt: str, call_name: str):
# Locate the LAST occurrence of `call_name(` in the file (plotly's JS
# bundle mentions Plotly.newPlot in docstrings/code earlier, but the real
# user-data call is always emitted at the very bottom).
start = txt.rfind(call_name + "(")
if start < 0:
raise ValueError(f"{call_name}( not found")
open_paren = txt.index("(", start)
i = open_paren + 1
depth = 1
in_str = False
escape = False
args: list[str] = []
cur = i
n = len(txt)
while i < n:
c = txt[i]
if in_str:
if escape:
escape = False
elif c == "\\":
escape = True
elif c == '"':
in_str = False
else:
if c == '"':
in_str = True
elif c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth == 0:
args.append(txt[cur:i].strip())
return args
elif c == "," and depth == 1:
args.append(txt[cur:i].strip())
cur = i + 1
i += 1
raise ValueError(f"unterminated {call_name} arglist")
def _parse_stem(stem: str) -> dict:
m = _STEM_RE.match(stem)
if not m:
raise ValueError(f"stem does not match expected pattern: {stem!r}")
return {
"stem": stem,
"generator": m.group("gen"),
"embedder": m.group("emb"),
"num_points": int(m.group("n")),
"num_timesteps": int(m.group("t")),
"jitter_scale": float(m.group("j")),
"seed": int(m.group("s")),
}
def _trace_xy_by_id(trace: dict) -> tuple[list[int], list[float], list[float]]:
# Prefer explicit `ids`; fall back to customdata (which in this flow also
# carries the point id in column 0); finally, fall back to positional index.
if "ids" in trace:
ids = _decode_bdata(trace["ids"])
# ids may arrive as strings too (plotly coerces). If they're strings
# of digits, we leave them as-is but prefer ints when possible.
if ids and isinstance(ids[0], str) and ids[0].isdigit():
ids = [int(v) for v in ids]
elif "customdata" in trace:
cd = trace["customdata"]
if isinstance(cd, dict):
ids = _decode_bdata(cd)
else:
# shape (N, k): pick first column
ids = [row[0] if isinstance(row, (list, tuple)) else row for row in cd]
else:
ids = list(range(len(_decode_bdata(trace["x"]))))
xs = _decode_bdata(trace["x"])
ys = _decode_bdata(trace["y"])
if not (len(ids) == len(xs) == len(ys)):
raise ValueError(
f"trace length mismatch: ids={len(ids)} x={len(xs)} y={len(ys)}"
)
return ids, xs, ys
def _frame_xy_ordered(traces: list[dict], point_ids: list[int]):
# Merge possibly multiple traces (one per label-group) back into a single
# (x, y) pair ordered by point_ids. Ids missing from this frame — points
# added or removed between frames by the jitter_add_remove path — become
# `None` so downstream code can skip them.
by_id: dict[int, tuple[float, float]] = {}
for t in traces:
ids, xs, ys = _trace_xy_by_id(t)
for pid, x, y in zip(ids, xs, ys):
by_id[pid] = (x, y)
xs_out = [by_id[pid][0] if pid in by_id else None for pid in point_ids]
ys_out = [by_id[pid][1] if pid in by_id else None for pid in point_ids]
return xs_out, ys_out
def _labels_for_ids(traces: list[dict], point_ids: list[int]) -> list[str]:
# If traces carry distinct names/legendgroups, use that as the per-point
# class label. When all traces share an empty name (the single-trace case
# emitted by this flow), labels are uniformly "".
names = [t.get("name") or t.get("legendgroup") or "" for t in traces]
if len(traces) <= 1 or all(n == names[0] for n in names):
return ["" for _ in point_ids]
by_id: dict[int, str] = {}
for t, name in zip(traces, names):
ids, _, _ = _trace_xy_by_id(t)
for pid in ids:
by_id[pid] = name
return [by_id.get(pid, "") for pid in point_ids]
def parse_plotly_run(html_path) -> dict:
"""Parse a flow-emitted plotly HTML into a frames dict suitable for
the three.js comparison page. Raises ValueError on unrecognised shape."""
path = Path(html_path)
stem = path.stem
meta = _parse_stem(stem)
txt = path.read_text(encoding="utf-8")
new_args = _extract_call_args(txt, "Plotly.newPlot")
if len(new_args) < 3:
raise ValueError(f"Plotly.newPlot expected >=3 args, got {len(new_args)}")
initial_traces = json.loads(new_args[1])
layout = json.loads(new_args[2])
add_args = _extract_call_args(txt, "Plotly.addFrames")
if len(add_args) < 2:
raise ValueError(f"Plotly.addFrames expected 2 args, got {len(add_args)}")
raw_frames = json.loads(add_args[1])
if not isinstance(raw_frames, list) or not raw_frames:
raise ValueError("Plotly.addFrames second arg is not a non-empty list")
# Establish the stable id set as the UNION of ids across every frame —
# the flow's jitter_add_remove path intentionally adds/removes points
# between frames, so no single frame is authoritative. Order is: first
# appearance across frames, then by numeric id within each frame's new ids.
seen_ids: list[int] = []
seen_set: set = set()
for fr in raw_frames:
for t in fr.get("data") or []:
ids, _, _ = _trace_xy_by_id(t)
for pid in ids:
if pid not in seen_set:
seen_set.add(pid)
seen_ids.append(pid)
if not seen_ids:
raise ValueError("no point ids recovered from any frame")
first_frame_traces = raw_frames[0].get("data") or initial_traces
labels = _labels_for_ids(first_frame_traces, seen_ids)
frames_out = []
times = []
for i, fr in enumerate(raw_frames):
traces = fr.get("data")
if not traces:
raise ValueError(f"frame {i} has no 'data'")
xs, ys = _frame_xy_ordered(traces, seen_ids)
frames_out.append({"x": xs, "y": ys})
times.append(fr.get("name", str(i)))
# Global bounds across all frames (skipping None placeholders for
# points that only exist in some frames).
all_x = [v for fr in frames_out for v in fr["x"] if v is not None]
all_y = [v for fr in frames_out for v in fr["y"] if v is not None]
if not all_x or not all_y:
raise ValueError("no finite x/y values across frames")
xmin, xmax = min(all_x), max(all_x)
ymin, ymax = min(all_y), max(all_y)
title = ""
try:
title = layout["title"]["text"] if isinstance(layout.get("title"), dict) else str(layout.get("title") or "")
except Exception:
title = ""
meta["title"] = title
return {
"meta": meta,
"point_ids": seen_ids,
"labels": labels,
"times": times,
"frames": frames_out,
"bounds": {"x": [xmin, xmax], "y": [ymin, ymax]},
}
if __name__ == "__main__":
import sys
import traceback
figs_dir = Path("/home/mm/work/dr-sandbox/figs")
html_files = sorted(figs_dir.glob("*.html"))
if not html_files:
print("no .html files in", figs_dir)
sys.exit(1)
failures = []
for p in html_files:
try:
out = parse_plotly_run(p)
except Exception as e:
failures.append((p.name, str(e)))
print(f"FAIL {p.name}: {e}")
traceback.print_exc()
continue
m = out["meta"]
pids = out["point_ids"]
frames = out["frames"]
nT = len(frames)
f0 = frames[0]
f0x = [v for v in f0["x"] if v is not None]
f0y = [v for v in f0["y"] if v is not None]
xr = (min(f0x), max(f0x)) if f0x else (float("nan"), float("nan"))
yr = (min(f0y), max(f0y)) if f0y else (float("nan"), float("nan"))
consistent = all(len(fr["x"]) == len(pids) and len(fr["y"]) == len(pids) for fr in frames)
present_per_frame = [sum(v is not None for v in fr["x"]) for fr in frames]
present_rng = (min(present_per_frame), max(present_per_frame))
print(
f"OK {m['stem']} |ids|={len(pids)} T={nT} "
f"present/frame=[{present_rng[0]},{present_rng[1]}] "
f"f0 x=[{xr[0]:+.3f},{xr[1]:+.3f}] y=[{yr[0]:+.3f},{yr[1]:+.3f}] "
f"consistent={consistent}"
)
print()
if failures:
print(f"{len(failures)} failure(s):")
for name, msg in failures:
print(f" {name}: {msg}")
else:
print(f"all {len(html_files)} files parsed OK")

64
app/web/static/compare-select.js
View File

@ -1,64 +0,0 @@
// Manages run-comparison selection on the runs list.
// HTMX re-renders #runs-slot every 3s, so we keep state in a Set outside
// the polled region and re-apply checked state on every afterSwap.
(function () {
const MAX = 2;
const selected = new Set();
const btn = document.getElementById('compare-btn');
const countEl = document.getElementById('compare-count');
const slot = document.getElementById('runs-slot');
if (!btn || !countEl || !slot) return;
function refreshButton() {
const n = selected.size;
countEl.textContent = `(${n}/${MAX})`;
btn.disabled = n !== MAX;
}
function applyToDOM() {
const checkboxes = slot.querySelectorAll('.compare-cb');
// Drop any selected stems that are no longer in the DOM (run aged out of list)
const present = new Set();
checkboxes.forEach((cb) => present.add(cb.dataset.stem));
for (const s of [...selected]) if (!present.has(s)) selected.delete(s);
const atCap = selected.size >= MAX;
checkboxes.forEach((cb) => {
const stem = cb.dataset.stem;
cb.checked = selected.has(stem);
cb.disabled = atCap && !cb.checked;
});
refreshButton();
}
slot.addEventListener('change', (e) => {
const cb = e.target;
if (!cb.matches || !cb.matches('.compare-cb')) return;
const stem = cb.dataset.stem;
if (cb.checked) {
if (selected.size >= MAX) {
cb.checked = false;
return;
}
selected.add(stem);
} else {
selected.delete(stem);
}
applyToDOM();
});
document.body.addEventListener('htmx:afterSwap', (e) => {
if (e.target && e.target.id === 'runs-slot') applyToDOM();
});
btn.addEventListener('click', () => {
if (selected.size !== MAX) return;
const [a, b] = [...selected];
const url = `/compare?a=${encodeURIComponent(a)}&b=${encodeURIComponent(b)}`;
window.open(url, '_blank', 'noopener');
});
applyToDOM();
})();

733
app/web/static/compare.js
View File

@ -1,733 +0,0 @@
// compare.js — side-by-side animated scatter for two embedding runs.
//
// Reads ?a=<stemA>&b=<stemB> from the URL, fetches /api/runs/<stem>/frames.json
// for each, and renders them into two linked three.js panels with a shared
// play/scrub/speed control strip. Linked hover: cursor on a point in one
// panel highlights the same point_id in the other.
import * as THREE from 'three';
// -------- URL / DOM wiring ------------------------------------------------
const params = new URLSearchParams(window.location.search);
const STEM_A = params.get('a') || '';
const STEM_B = params.get('b') || '';
const layout = document.getElementById('compare-layout');
const panelElA = layout.querySelector('.compare-panel[data-slot="a"]');
const panelElB = layout.querySelector('.compare-panel[data-slot="b"]');
const playBtn = document.getElementById('cc-play');
const scrub = document.getElementById('cc-scrub');
const speedSel = document.getElementById('cc-speed');
const syncSel = document.getElementById('cc-sync');
const motionSel = document.getElementById('cc-motion');
const colorSel = document.getElementById('cc-color');
const timeAEl = document.getElementById('cc-time').querySelector('[data-role="time-a"]');
const timeBEl = document.getElementById('cc-time').querySelector('[data-role="time-b"]');
// -------- small helpers ---------------------------------------------------
// Build a soft-edged circular sprite for THREE.Points. A plain texture with
// radial alpha gives anti-aliased round dots without fragment-shader work.
function makeDiskTexture(size = 64) {
const c = document.createElement('canvas');
c.width = c.height = size;
const g = c.getContext('2d');
const r = size / 2;
const grd = g.createRadialGradient(r, r, 0, r, r, r);
grd.addColorStop(0.0, 'rgba(255,255,255,1)');
grd.addColorStop(0.55, 'rgba(255,255,255,1)');
grd.addColorStop(0.85, 'rgba(255,255,255,0.35)');
grd.addColorStop(1.0, 'rgba(255,255,255,0)');
g.fillStyle = grd;
g.fillRect(0, 0, size, size);
const tex = new THREE.CanvasTexture(c);
tex.needsUpdate = true;
return tex;
}
const DISK_TEX = makeDiskTexture();
// Continuous ramp (blue → orange), matching dataset-picker.js exactly.
function rampContinuous(t, out) {
const hue = (1 - t) * 215 + t * 28;
const sat = 0.62;
const lit = 0.50 + (t - 0.5) * 0.08;
return (out || new THREE.Color()).setHSL(hue / 360, sat, lit);
}
// 8-color categorical palette — same hex list as dataset-picker.js.
const CATEGORICAL_HEX = [
'#1f4e5f', '#c97b3f', '#8b5a9f', '#5a8560',
'#c74a5e', '#6b7d8f', '#b89f51', '#4a6fa5',
];
const CATEGORICAL = CATEGORICAL_HEX.map(h => new THREE.Color(h));
// Precompute per-point RGB, indexed by position in data.point_ids.
// If the server attached labels + label_kind, color by that (ramp for
// continuous, palette for categorical) to match the dataset picker. Points
// with null labels (jitter-added, id >= num_points) stay (0,0,0) = black.
// Falls back to a frame-0-present ordinal ramp when no labels are present.
function buildIdColorsRGB(data) {
const n = data.point_ids.length;
const rgb = new Float32Array(n * 3);
const labels = data.labels || [];
const kind = data.label_kind || null;
const hasRealLabels = kind && labels.some(v => v != null && v !== '');
if (hasRealLabels) {
const tmp = new THREE.Color();
if (kind === 'categorical') {
for (let i = 0; i < n; i++) {
const v = labels[i];
if (v == null) continue;
const c = CATEGORICAL[((v | 0) % CATEGORICAL.length + CATEGORICAL.length) % CATEGORICAL.length];
rgb[i * 3 + 0] = c.r;
rgb[i * 3 + 1] = c.g;
rgb[i * 3 + 2] = c.b;
}
} else {
let lo = Infinity, hi = -Infinity;
for (const v of labels) { if (v == null) continue; if (v < lo) lo = v; if (v > hi) hi = v; }
const range = (hi - lo) || 1;
for (let i = 0; i < n; i++) {
const v = labels[i];
if (v == null) continue;
rampContinuous((v - lo) / range, tmp);
rgb[i * 3 + 0] = tmp.r;
rgb[i * 3 + 1] = tmp.g;
rgb[i * 3 + 2] = tmp.b;
}
}
return rgb;
}
// Fallback: rainbow-by-ordinal over frame-0-present points.
const frame0 = data.frames[0];
if (!frame0) return rgb;
const originalPositions = [];
for (let i = 0; i < n; i++) {
if (frame0.x[i] != null && frame0.y[i] != null
&& !Number.isNaN(frame0.x[i]) && !Number.isNaN(frame0.y[i])) {
originalPositions.push(i);
}
}
originalPositions.sort((a, b) => data.point_ids[a] - data.point_ids[b]);
const nOrig = originalPositions.length;
const tmp = new THREE.Color();
for (let k = 0; k < nOrig; k++) {
const t = nOrig > 1 ? k / (nOrig - 1) : 0.5;
rampContinuous(t, tmp);
const idx = originalPositions[k];
rgb[idx * 3 + 0] = tmp.r;
rgb[idx * 3 + 1] = tmp.g;
rgb[idx * 3 + 2] = tmp.b;
}
return rgb;
}
function readVar(name, fallback) {
const v = getComputedStyle(document.documentElement).getPropertyValue(name).trim();
return v || fallback;
}
function readPanelBg(el) {
const v = getComputedStyle(el).getPropertyValue('--picker-panel').trim();
return v || readVar('--panel', '#ffffff');
}
// Panel accent colors keyed by slot. Resolved from CSS vars so they flip
// with the theme via 'themechange'.
function panelAccent(slot) {
return slot === 'a' ? readVar('--accent', '#1f4e5f') : readVar('--warm', '#a77a2c');
}
function highlightColor() {
return readVar('--alarm', '#8a3a2a');
}
// -------- Panel factory ---------------------------------------------------
// Returns { setFrame, setBounds, setHighlight, onHover, resize, dispose, state }
function createPanel({ slotId, panelEl, data }) {
const canvasEl = panelEl.querySelector('[data-role="canvas"]');
const statusEl = panelEl.querySelector('[data-role="status"]');
const scene = new THREE.Scene();
scene.background = new THREE.Color(readPanelBg(panelEl));
const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, -10, 10);
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: false });
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
canvasEl.appendChild(renderer.domElement);
// Pre-allocate a buffer sized for num_points. Each frame we repack the
// non-null points into the prefix and call setDrawRange(0, count).
const maxN = data.point_ids.length;
const positions = new Float32Array(maxN * 3);
const colors = new Float32Array(maxN * 3);
const ids = new Int32Array(maxN); // packed point_id per drawn vertex
const idColorRGB = buildIdColorsRGB(data);
const geo = new THREE.BufferGeometry();
geo.setAttribute('position', new THREE.BufferAttribute(positions, 3));
geo.setAttribute('color', new THREE.BufferAttribute(colors, 3));
geo.setDrawRange(0, 0);
const matMono = new THREE.PointsMaterial({
size: 6.0,
sizeAttenuation: false,
map: DISK_TEX,
transparent: true,
alphaTest: 0.2,
depthWrite: false,
color: new THREE.Color(panelAccent(slotId)),
});
const matRainbow = new THREE.PointsMaterial({
size: 6.0,
sizeAttenuation: false,
map: DISK_TEX,
// Opaque + alpha cutoff rather than alpha blending: with a ramp, many
// overlapping translucent points average out to the middle of the ramp
// (blue + orange → green), washing out the whole cloud. Hard edges keep
// each point's true color visible.
transparent: false,
alphaTest: 0.5,
vertexColors: true,
});
let mat = matMono;
const points = new THREE.Points(geo, mat);
scene.add(points);
// Highlight overlay — a second 1-vertex Points object drawn on top.
const hiPos = new Float32Array(3);
const hiGeo = new THREE.BufferGeometry();
hiGeo.setAttribute('position', new THREE.BufferAttribute(hiPos, 3));
hiGeo.setDrawRange(0, 0);
const hiMat = new THREE.PointsMaterial({
size: 14.0,
sizeAttenuation: false,
map: DISK_TEX,
transparent: true,
alphaTest: 0.05,
depthWrite: false,
color: new THREE.Color(highlightColor()),
opacity: 0.9,
});
const hiPoints = new THREE.Points(hiGeo, hiMat);
// Ensure highlight renders above base points
hiPoints.renderOrder = 1;
scene.add(hiPoints);
// ---- current-frame packed state (kept in closure for hover picking) ----
// packedX/packedY/packedId of length = current draw count.
let packedN = 0;
const packedX = new Float32Array(maxN);
const packedY = new Float32Array(maxN);
const packedId = new Int32Array(maxN);
// Camera frame rectangle (world coords) currently applied.
const camRect = { xmin: -1, xmax: 1, ymin: -1, ymax: 1 };
function applyCamRect(rect) {
const pad = 0.05;
const dx = rect.xmax - rect.xmin || 1;
const dy = rect.ymax - rect.ymin || 1;
const cx = (rect.xmin + rect.xmax) / 2;
const cy = (rect.ymin + rect.ymax) / 2;
const rx = dx * (0.5 + pad);
const ry = dy * (0.5 + pad);
// Fit-to-larger-axis so points never get squashed when the panel aspect
// doesn't match the bounds aspect. We compute the viewport aspect here
// so the ortho frustum covers the data and then some.
const rect2 = canvasEl.getBoundingClientRect();
const vw = Math.max(1, rect2.width);
const vh = Math.max(1, rect2.height);
const aspect = vw / vh;
const dataAspect = (rx * 2) / (ry * 2);
let halfW, halfH;
if (aspect > dataAspect) {
// viewport wider than data: expand X to fit
halfH = ry;
halfW = ry * aspect;
} else {
halfW = rx;
halfH = rx / aspect;
}
camera.left = cx - halfW;
camera.right = cx + halfW;
camera.top = cy + halfH;
camera.bottom = cy - halfH;
camera.updateProjectionMatrix();
camRect.xmin = rect.xmin;
camRect.xmax = rect.xmax;
camRect.ymin = rect.ymin;
camRect.ymax = rect.ymax;
}
function resize() {
const rect = canvasEl.getBoundingClientRect();
const w = Math.max(1, Math.floor(rect.width));
const h = Math.max(1, Math.floor(rect.height));
renderer.setSize(w, h, false);
// Re-apply the current cam rect so the aspect fit recomputes.
applyCamRect(camRect);
}
function setBounds(b) {
applyCamRect({ xmin: b.x[0], xmax: b.x[1], ymin: b.y[0], ymax: b.y[1] });
}
function applyColorsFromTheme() {
matMono.color.set(panelAccent(slotId));
hiMat.color.set(highlightColor());
scene.background = new THREE.Color(readPanelBg(panelEl));
}
function setColorMode(mode) {
const next = mode === 'mono' ? matMono : matRainbow;
if (points.material !== next) {
points.material = next;
mat = next;
}
}
// Copy the precomputed per-id RGB into the packed position `j`.
function writePackedColor(j, i) {
colors[j * 3 + 0] = idColorRGB[i * 3 + 0];
colors[j * 3 + 1] = idColorRGB[i * 3 + 1];
colors[j * 3 + 2] = idColorRGB[i * 3 + 2];
}
// Pack frame `f` into the geometry buffer, skipping null x/y.
function setFrame(f) {
const frame = data.frames[f];
if (!frame) return;
const xs = frame.x, ys = frame.y;
const ptIds = data.point_ids;
let j = 0;
for (let i = 0; i < xs.length; i++) {
const x = xs[i], y = ys[i];
if (x === null || y === null || x === undefined || y === undefined
|| Number.isNaN(x) || Number.isNaN(y)) continue;
positions[j * 3 + 0] = x;
positions[j * 3 + 1] = y;
positions[j * 3 + 2] = 0;
writePackedColor(j, i);
packedX[j] = x;
packedY[j] = y;
packedId[j] = ptIds[i];
j++;
}
packedN = j;
geo.attributes.position.needsUpdate = true;
geo.attributes.color.needsUpdate = true;
geo.setDrawRange(0, packedN);
applyHighlightForCurrentFrame();
}
// Pack an interpolated frame. uLocal is a continuous index in [0, T-1].
// Points missing in either adjacent frame are skipped for the duration of
// that transition (no connect-back to the last-known position).
function setFrameInterpolated(uLocal) {
const T = data.frames.length;
if (T === 0) return;
if (uLocal <= 0) return setFrame(0);
if (uLocal >= T - 1) return setFrame(T - 1);
const f0 = Math.floor(uLocal);
const f1 = f0 + 1;
const t = uLocal - f0;
const fr0 = data.frames[f0], fr1 = data.frames[f1];
const x0 = fr0.x, y0 = fr0.y, x1 = fr1.x, y1 = fr1.y;
const ptIds = data.point_ids;
let j = 0;
for (let i = 0; i < x0.length; i++) {
const a = x0[i], b = x1[i], c = y0[i], d = y1[i];
if (a === null || a === undefined || Number.isNaN(a)
|| b === null || b === undefined || Number.isNaN(b)
|| c === null || c === undefined || Number.isNaN(c)
|| d === null || d === undefined || Number.isNaN(d)) continue;
const xi = a + (b - a) * t;
const yi = c + (d - c) * t;
positions[j * 3 + 0] = xi;
positions[j * 3 + 1] = yi;
positions[j * 3 + 2] = 0;
writePackedColor(j, i);
packedX[j] = xi;
packedY[j] = yi;
packedId[j] = ptIds[i];
j++;
}
packedN = j;
geo.attributes.position.needsUpdate = true;
geo.attributes.color.needsUpdate = true;
geo.setDrawRange(0, packedN);
applyHighlightForCurrentFrame();
}
// ---- highlight by point_id ---------------------------------------------
let currentHighlightId = -1;
function applyHighlightForCurrentFrame() {
if (currentHighlightId < 0) {
hiGeo.setDrawRange(0, 0);
return;
}
// Linear scan — packedN <= 5000 and this only runs on hover.
for (let i = 0; i < packedN; i++) {
if (packedId[i] === currentHighlightId) {
hiPos[0] = packedX[i];
hiPos[1] = packedY[i];
hiPos[2] = 0.01;
hiGeo.attributes.position.needsUpdate = true;
hiGeo.setDrawRange(0, 1);
return;
}
}
// Not present this frame.
hiGeo.setDrawRange(0, 0);
}
function setHighlight(pointId) {
currentHighlightId = (pointId === null || pointId === undefined) ? -1 : pointId;
applyHighlightForCurrentFrame();
}
// Convert clientX/Y to world coords (ortho, no rotation).
function clientToWorld(clientX, clientY) {
const rect = canvasEl.getBoundingClientRect();
const u = (clientX - rect.left) / Math.max(1, rect.width);
const v = (clientY - rect.top) / Math.max(1, rect.height);
const wx = camera.left + u * (camera.right - camera.left);
const wy = camera.top - v * (camera.top - camera.bottom);
return { wx, wy };
}
// Nearest-point pick with a screen-space radius cap. Returns point_id or -1.
function pickAt(clientX, clientY) {
if (packedN === 0) return -1;
const { wx, wy } = clientToWorld(clientX, clientY);
// Screen-pixel radius -> world radius
const rect = canvasEl.getBoundingClientRect();
const worldPerPx = (camera.right - camera.left) / Math.max(1, rect.width);
const pickPx = 14;
const maxR = pickPx * worldPerPx;
const maxR2 = maxR * maxR;
let bestI = -1;
let bestD2 = Infinity;
for (let i = 0; i < packedN; i++) {
const dx = packedX[i] - wx;
const dy = packedY[i] - wy;
const d2 = dx * dx + dy * dy;
if (d2 < bestD2 && d2 < maxR2) {
bestD2 = d2;
bestI = i;
}
}
return bestI < 0 ? -1 : packedId[bestI];
}
function render() {
renderer.render(scene, camera);
}
function dispose() {
geo.dispose();
hiGeo.dispose();
matMono.dispose();
matRainbow.dispose();
hiMat.dispose();
renderer.dispose();
if (renderer.domElement.parentNode) {
renderer.domElement.parentNode.removeChild(renderer.domElement);
}
}
// Hide the status overlay once we have data.
statusEl.style.display = 'none';
return {
slotId,
canvasEl,
panelEl,
data,
setFrame,
setFrameInterpolated,
setColorMode,
setBounds,
setHighlight,
resize,
render,
pickAt,
applyColorsFromTheme,
dispose,
get packedN() { return packedN; },
get camRect() { return camRect; },
applyCamRect,
};
}
// -------- error rendering -------------------------------------------------
function renderError(panelEl, stem, msg) {
const statusEl = panelEl.querySelector('[data-role="status"]');
statusEl.style.display = '';
statusEl.classList.add('is-error');
statusEl.textContent = `could not load ${stem}: ${msg}`;
// Keep header readable
panelEl.querySelector('[data-role="embedder"]').textContent = '—';
panelEl.querySelector('[data-role="generator"]').textContent = '—';
panelEl.querySelector('[data-role="params"]').textContent = '(error)';
}
const PARAM_FIELDS = [
{ key: 'num_points', prefix: 'N' },
{ key: 'num_timesteps', prefix: 'T' },
{ key: 'jitter_scale', prefix: 'J' },
{ key: 'seed', prefix: 's' },
];
function renderHeader(panelEl, data) {
const m = data.meta || {};
panelEl.querySelector('[data-role="embedder"]').textContent = m.embedder || '—';
panelEl.querySelector('[data-role="generator"]').textContent = m.generator || '—';
const host = panelEl.querySelector('[data-role="params"]');
host.textContent = '';
PARAM_FIELDS.forEach(({ key, prefix }, i) => {
if (i > 0) host.appendChild(document.createTextNode(' / '));
const span = document.createElement('span');
span.className = 'param';
span.dataset.key = key;
span.textContent = `${prefix}${m[key] ?? '?'}`;
host.appendChild(span);
});
}
// Toggle .diff on header spans where the two panels disagree.
function markParamDiffs(metaA, metaB) {
if (!metaA || !metaB) return;
for (const { key } of PARAM_FIELDS) {
const differs = metaA[key] !== metaB[key];
for (const panelEl of [panelElA, panelElB]) {
const span = panelEl.querySelector(`[data-role="params"] .param[data-key="${key}"]`);
if (span) span.classList.toggle('diff', differs);
}
}
for (const role of ['embedder', 'generator']) {
const differs = metaA[role] !== metaB[role];
for (const panelEl of [panelElA, panelElB]) {
const span = panelEl.querySelector(`[data-role="${role}"]`);
if (span) span.classList.toggle('diff', differs);
}
}
}
// -------- main ------------------------------------------------------------
async function fetchFrames(stem) {
const res = await fetch(`/api/runs/${encodeURIComponent(stem)}/frames.json`, { cache: 'no-store' });
if (!res.ok) {
throw new Error(`${res.status} ${res.statusText}`);
}
return res.json();
}
async function main() {
if (!STEM_A || !STEM_B) {
renderError(panelElA, STEM_A || '(missing)', 'no stem in ?a=');
renderError(panelElB, STEM_B || '(missing)', 'no stem in ?b=');
return;
}
// Fetch in parallel; each panel's failure is independent.
const [resA, resB] = await Promise.allSettled([
fetchFrames(STEM_A),
fetchFrames(STEM_B),
]);
const panels = { a: null, b: null };
if (resA.status === 'fulfilled') {
renderHeader(panelElA, resA.value);
panels.a = createPanel({ slotId: 'a', panelEl: panelElA, data: resA.value });
} else {
renderError(panelElA, STEM_A, resA.reason?.message || String(resA.reason));
}
if (resB.status === 'fulfilled') {
renderHeader(panelElB, resB.value);
panels.b = createPanel({ slotId: 'b', panelEl: panelElB, data: resB.value });
} else {
renderError(panelElB, STEM_B, resB.reason?.message || String(resB.reason));
}
if (panels.a && panels.b) markParamDiffs(panels.a.data.meta, panels.b.data.meta);
// Nothing loaded — no animation loop to start.
if (!panels.a && !panels.b) return;
// Initial bounds + first frame for each panel.
for (const p of Object.values(panels)) {
if (!p) continue;
p.setBounds(p.data.bounds);
p.resize();
p.setFrame(0);
}
// ---- time mapping -----------------------------------------------------
// Scrubber is 0..1000, mapped to a shared global frame index uGlobal in
// [0, maxT-1]. Each panel clamps uGlobal to its own (T-1) — so a shorter
// timeline pads its last frame until the longer timeline finishes, and
// both panels advance at the same wall-clock tempo.
const SCRUB_MAX = 1000;
function framesOf(p) { return p ? p.data.frames.length : 0; }
function timeLabelForAtLocal(p, uLocal) {
if (!p) return '—';
const T = framesOf(p);
if (T <= 0) return '—';
const idx = Math.max(0, Math.min(T - 1, Math.round(uLocal)));
return p.data.times?.[idx] ?? String(idx);
}
function applyU(u) {
u = Math.max(0, Math.min(1, u));
const smooth = motionSel.value === 'smooth';
const tMax = maxT();
const uGlobal = u * (tMax - 1);
let uLocalA = 0, uLocalB = 0;
for (const [slot, p] of Object.entries(panels)) {
if (!p) continue;
const T = framesOf(p);
if (T <= 0) continue;
const uLocal = Math.min(uGlobal, T - 1);
if (slot === 'a') uLocalA = uLocal; else uLocalB = uLocal;
if (smooth) {
p.setFrameInterpolated(uLocal);
} else {
const idx = Math.max(0, Math.min(T - 1, Math.round(uLocal)));
p.setFrame(idx);
}
}
timeAEl.textContent = timeLabelForAtLocal(panels.a, uLocalA);
timeBEl.textContent = timeLabelForAtLocal(panels.b, uLocalB);
}
// ---- axes sync mode ---------------------------------------------------
function applySync() {
const mode = syncSel.value;
if (mode === 'locked' && panels.a && panels.b) {
const ba = panels.a.data.bounds, bb = panels.b.data.bounds;
const union = {
x: [Math.min(ba.x[0], bb.x[0]), Math.max(ba.x[1], bb.x[1])],
y: [Math.min(ba.y[0], bb.y[0]), Math.max(ba.y[1], bb.y[1])],
};
panels.a.setBounds(union);
panels.b.setBounds(union);
} else {
if (panels.a) panels.a.setBounds(panels.a.data.bounds);
if (panels.b) panels.b.setBounds(panels.b.data.bounds);
}
}
syncSel.addEventListener('change', applySync);
applySync();
// ---- play loop --------------------------------------------------------
// Base step: 400ms per frame at 1x, divided by speed. The loop advances
// the scrub by (1 / maxT) per step so both panels traverse their whole
// timeline in the same wall-clock duration when T differs.
let playing = false;
let lastTs = 0;
function maxT() {
const ta = framesOf(panels.a);
const tb = framesOf(panels.b);
return Math.max(ta, tb, 2);
}
function baseMsPerFrame() { return 1600 / parseFloat(speedSel.value || '1'); }
// u is the authoritative playhead (float in [0, 1]). The scrubber mirrors
// it for display; reading u back from the scrubber would quantize to the
// scrubber's integer step and stall the loop at small du.
let u = 0;
function tick(ts) {
requestAnimationFrame(tick);
for (const p of Object.values(panels)) p?.render();
if (!playing) { lastTs = ts; return; }
if (!lastTs) { lastTs = ts; return; }
const dt = ts - lastTs;
lastTs = ts;
if (dt <= 0) return;
const perFrame = baseMsPerFrame();
const T = maxT();
u += dt / (perFrame * (T - 1));
if (u >= 1) u -= Math.floor(u); // wrap 0..1
scrub.value = String(Math.round(u * SCRUB_MAX));
applyU(u);
}
requestAnimationFrame(tick);
function setPlaying(v) {
playing = v;
lastTs = 0;
playBtn.textContent = playing ? '▮▮' : '▶';
playBtn.setAttribute('aria-label', playing ? 'pause' : 'play');
}
playBtn.addEventListener('click', () => setPlaying(!playing));
scrub.addEventListener('input', () => {
u = parseFloat(scrub.value) / SCRUB_MAX;
applyU(u);
});
speedSel.addEventListener('change', () => { lastTs = 0; });
motionSel.addEventListener('change', () => {
applyU(parseFloat(scrub.value) / SCRUB_MAX);
});
function applyColorMode() {
const mode = colorSel.value;
for (const p of Object.values(panels)) p?.setColorMode(mode);
}
colorSel.addEventListener('change', applyColorMode);
applyColorMode();
// ---- linked hover -----------------------------------------------------
function wireHover(pA, pB) {
if (!pA) return;
const el = pA.canvasEl;
el.addEventListener('mousemove', (ev) => {
const id = pA.pickAt(ev.clientX, ev.clientY);
pA.setHighlight(id >= 0 ? id : null);
if (pB) pB.setHighlight(id >= 0 ? id : null);
});
el.addEventListener('mouseleave', () => {
pA.setHighlight(null);
if (pB) pB.setHighlight(null);
});
}
wireHover(panels.a, panels.b);
wireHover(panels.b, panels.a);
// ---- resize + theme ---------------------------------------------------
const ro = new ResizeObserver(() => {
for (const p of Object.values(panels)) p?.resize();
});
if (panels.a) ro.observe(panels.a.canvasEl);
if (panels.b) ro.observe(panels.b.canvasEl);
document.addEventListener('themechange', () => {
for (const p of Object.values(panels)) p?.applyColorsFromTheme();
});
// Initialise the label + scrub at 0.
applyU(0);
}
main().catch((err) => {
console.error(err);
renderError(panelElA, STEM_A, 'init failed: ' + err.message);
renderError(panelElB, STEM_B, 'init failed: ' + err.message);
});

306
app/web/static/style.css
View File

@ -344,25 +344,6 @@ select {
color: var(--faint);
font-size: 0.72rem;
align-self: start;
text-align: right;
}
/* Narrow viewports: let the blurb span full width under the name/pkg row
instead of getting crammed into the middle column. */
@media (max-width: 780px) {
.reducer-list label {
grid-template-columns: 1.4rem 1fr auto;
grid-template-areas:
"mark name pkg"
"mark blurb blurb";
column-gap: 0.6rem;
row-gap: 2px;
}
.reducer-list .mark { grid-area: mark; }
.reducer-list .pkg { grid-area: pkg; }
.reducer-list .text { display: contents; }
.reducer-list .name { grid-area: name; }
.reducer-list .blurb { grid-area: blurb; }
}
/* advanced disclosure */
@ -456,77 +437,15 @@ button.submit:disabled { background: var(--faint); border-color: var(--faint); c
border-bottom: 1px solid var(--rule);
padding: 0.85rem 0 0.85rem;
display: grid;
grid-template-columns: 1.25rem 5.5rem 1fr;
column-gap: 0.6rem;
grid-template-columns: 5.5rem 1fr;
column-gap: 1rem;
align-items: start;
}
.runs li.run > .compare-cb,
.runs li.run > .compare-cb-slot {
width: 1rem;
height: 1rem;
margin: 3px 0 0 0;
align-self: start;
cursor: pointer;
accent-color: var(--accent);
}
.runs li.run > .compare-cb-slot {
visibility: hidden;
cursor: default;
}
.runs li.run > .compare-cb:disabled {
cursor: not-allowed;
opacity: 0.35;
}
.compare-bar {
display: flex;
align-items: center;
gap: 0.7rem;
margin-bottom: 0.5rem;
flex-wrap: wrap;
}
.compare-bar button {
font: inherit;
font-size: 0.82rem;
background: transparent;
border: 1px solid var(--rule);
color: var(--ink);
padding: 0.3rem 0.7rem;
cursor: pointer;
transition: background 100ms ease, border-color 100ms ease;
}
.compare-bar button:not(:disabled):hover {
background: var(--accent-tint);
border-color: var(--accent);
}
.compare-bar button:disabled {
opacity: 0.45;
cursor: not-allowed;
}
.compare-bar #compare-count {
font-family: var(--mono);
font-size: 0.76rem;
color: var(--faint);
margin-left: 0.25rem;
}
.compare-hint {
font-size: 0.76rem;
font-style: italic;
}
.runs li.run.just-submitted {
background: linear-gradient(to right, var(--accent-tint), transparent 60%);
padding-left: 0.55rem;
margin-left: -0.55rem;
}
.runs li.run.stale {
opacity: 0.55;
}
.runs li.run.stale .stale-note {
color: var(--alarm);
font-style: italic;
font-size: 0.72rem;
margin-left: 0.3rem;
}
.run .stamp {
font-family: var(--mono);
@ -1460,224 +1379,3 @@ button.submit:disabled { background: var(--faint); border-color: var(--faint); c
.legend .legend-row { grid-template-columns: 10px 1fr; }
.legend .fn { display: none; }
}
/* ---------- compare page ------------------------------------------------ */
.compare-layout {
--panel-h: 62vh;
padding: 1.2rem 1.6rem 1.6rem;
display: flex;
flex-direction: column;
gap: 1rem;
}
.compare-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1rem;
}
.compare-panel {
display: flex;
flex-direction: column;
border: 1px solid var(--rule);
background: var(--panel);
border-radius: 2px;
min-width: 0;
min-height: 0;
overflow: hidden;
}
.compare-panel-head {
display: flex;
align-items: baseline;
gap: 0.55rem;
padding: 0.55rem 0.85rem;
border-bottom: 1px solid var(--rule);
font-family: var(--mono);
font-size: 0.78rem;
color: var(--mute);
flex-wrap: wrap;
}
.compare-panel-head .panel-tag {
display: inline-block;
min-width: 1.2em;
padding: 0 0.35rem;
border-radius: 2px;
background: var(--accent-tint);
color: var(--accent);
font-weight: 600;
text-align: center;
font-size: 0.72rem;
letter-spacing: 0.04em;
}
.compare-panel[data-slot="b"] .compare-panel-head .panel-tag {
color: var(--warm);
background: color-mix(in srgb, var(--warm) 14%, transparent);
}
.compare-panel-head .panel-embedder {
color: var(--ink);
font-weight: 600;
}
.compare-panel-head .panel-generator {
color: var(--ink);
}
.compare-panel-head .panel-sep {
color: var(--faint);
}
.compare-panel-head .panel-params {
color: var(--mute);
font-size: 0.74rem;
flex-basis: 100%;
}
.compare-panel-head .panel-params .param.diff,
.compare-panel-head .panel-embedder.diff,
.compare-panel-head .panel-generator.diff {
color: var(--alarm);
font-weight: 600;
}
.compare-panel-head .panel-stem {
margin-left: auto;
color: var(--faint);
font-size: 0.72rem;
border-bottom: 1px solid transparent;
max-width: 100%;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.compare-panel-head .panel-stem:hover {
color: var(--mute);
border-bottom-color: var(--faint);
}
.compare-canvas {
position: relative;
flex: 1 1 auto;
min-height: 0;
background: var(--picker-panel, var(--panel));
height: var(--panel-h);
}
.compare-canvas canvas {
display: block;
width: 100% !important;
height: 100% !important;
}
.compare-status {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
padding: 1rem;
text-align: center;
color: var(--mute);
font-family: var(--mono);
font-size: 0.82rem;
pointer-events: none;
}
.compare-status.is-error {
color: var(--alarm);
}
.compare-controls {
display: flex;
align-items: center;
gap: 0.9rem;
padding: 0.7rem 0.9rem;
border: 1px solid var(--rule);
border-radius: 2px;
background: var(--panel);
font-family: var(--mono);
font-size: 0.8rem;
color: var(--mute);
flex-wrap: wrap;
}
.compare-controls .cc-play {
appearance: none;
border: 1px solid var(--rule-2);
background: var(--panel);
color: var(--ink);
width: 2.1rem;
height: 2.1rem;
border-radius: 2px;
cursor: pointer;
font-size: 0.9rem;
line-height: 1;
display: inline-flex;
align-items: center;
justify-content: center;
transition: background 120ms ease, border-color 120ms ease;
}
.compare-controls .cc-play:hover {
background: var(--accent-tint);
border-color: var(--accent);
color: var(--accent);
}
.compare-controls .cc-scrub {
flex: 1 1 auto;
min-width: 10rem;
accent-color: var(--accent);
}
.compare-controls .cc-time {
display: inline-flex;
gap: 0.35rem;
white-space: nowrap;
color: var(--ink);
font-variant-numeric: tabular-nums;
}
.compare-controls .cc-time-sep { color: var(--faint); }
.compare-controls .cc-time-a { color: var(--accent); }
.compare-controls .cc-time-b { color: var(--warm); }
.compare-controls .cc-speed-wrap,
.compare-controls .cc-sync-wrap,
.compare-controls .cc-motion-wrap,
.compare-controls .cc-color-wrap {
display: inline-flex;
align-items: center;
gap: 0.35rem;
}
.compare-controls .cc-lbl {
color: var(--faint);
text-transform: lowercase;
font-size: 0.74rem;
}
.compare-controls select {
appearance: none;
-webkit-appearance: none;
border: 1px solid var(--rule-2);
background: var(--panel);
color: var(--ink);
font-family: var(--mono);
font-size: 0.78rem;
padding: 0.25rem 1.4rem 0.25rem 0.5rem;
border-radius: 2px;
cursor: pointer;
background-image: linear-gradient(45deg, transparent 50%, var(--mute) 50%),
linear-gradient(-45deg, transparent 50%, var(--mute) 50%);
background-position: calc(100% - 12px) 50%, calc(100% - 7px) 50%;
background-size: 5px 5px, 5px 5px;
background-repeat: no-repeat;
}
.compare-controls select:focus {
outline: none;
border-color: var(--accent);
}
@media (max-width: 900px) {
.compare-layout { --panel-h: 45vh; padding: 1rem 0.9rem 1.2rem; }
.compare-grid { grid-template-columns: 1fr; }
}

8
app/web/templates/_runs.html
View File

@ -7,12 +7,7 @@
{% else %}
<ul class="runs">
{% for r in runs %}
<li class="run {% if just_submitted is defined and r.id == just_submitted %}just-submitted{% endif %}{% if r.stale %} stale{% endif %}">
{% if r.emb_exists and not r.stale %}
<input type="checkbox" class="compare-cb" data-stem="{{ r.emb_file[:-5] }}" aria-label="select run for comparison" />
{% else %}
<span class="compare-cb-slot" aria-hidden="true"></span>
{% endif %}
<li class="run {% if just_submitted is defined and r.id == just_submitted %}just-submitted{% endif %}">
<div class="stamp">
{% if r.runtime %}<span class="rt">{{ r.runtime }}</span>{% endif %}
{% if r.start %}{{ r.start[:10] }}<br/>{{ r.start[11:19] }}{% else %}&nbsp;{% endif %}
@ -47,7 +42,6 @@
<div class="outputs">
<span class="tag">fig</span>
{% if r.stale %}<span class="stale-note" title="a newer run with identical params overwrote this output">overwritten</span>{% endif %}
{% if r.ref_file %}
{% if r.ref_exists %}
<a href="/figs/{{ r.ref_file }}" target="_blank" rel="noopener">reference</a>

124
app/web/templates/compare.html
View File

@ -1,124 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title>embedding notebook &middot; compare</title>
<link rel="stylesheet" href="/static/style.css?v=27" />
<script type="importmap">
{
"imports": {
"three": "https://unpkg.com/three@0.160.0/build/three.module.js",
"three/addons/": "https://unpkg.com/three@0.160.0/examples/jsm/"
}
}
</script>
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Ccircle cx='8' cy='8' r='3' fill='%231f4e5f'/%3E%3C/svg%3E" />
<script>
(function(){try{
var t=localStorage.getItem('theme');
if(!t)t=matchMedia('(prefers-color-scheme: dark)').matches?'dark':'light';
document.documentElement.setAttribute('data-theme',t);
}catch(e){}})();
</script>
</head>
<body>
<header class="masthead">
<div>
<h1 class="title">embedding notebook <em>&mdash; compare</em></h1>
</div>
<div class="meta">
<a href="/" class="masthead-link">&larr; runs</a>
<button type="button" class="theme-toggle" id="theme-toggle" aria-label="toggle theme">&#9680;</button>
</div>
</header>
<section class="compare-layout" id="compare-layout"
data-stem-a="{{ stem_a }}" data-stem-b="{{ stem_b }}">
<div class="compare-controls" id="compare-controls">
<button type="button" class="cc-play" id="cc-play" aria-label="play / pause">&#9654;</button>
<input class="cc-scrub" id="cc-scrub" type="range"
min="0" max="1000" step="1" value="0" aria-label="time scrubber" />
<div class="cc-time" id="cc-time">
<span class="cc-time-a" data-role="time-a">&mdash;</span>
<span class="cc-time-sep">/</span>
<span class="cc-time-b" data-role="time-b">&mdash;</span>
</div>
<label class="cc-speed-wrap">
<span class="cc-lbl">speed</span>
<select class="cc-speed" id="cc-speed">
<option value="0.5">0.5&times;</option>
<option value="1" selected>1&times;</option>
<option value="2">2&times;</option>
<option value="4">4&times;</option>
</select>
</label>
<label class="cc-motion-wrap">
<span class="cc-lbl">motion</span>
<select class="cc-motion" id="cc-motion">
<option value="smooth" selected>smooth</option>
<option value="step">step</option>
</select>
</label>
<label class="cc-color-wrap">
<span class="cc-lbl">color</span>
<select class="cc-color" id="cc-color">
<option value="mono">mono</option>
<option value="original" selected>original</option>
</select>
</label>
<label class="cc-sync-wrap">
<span class="cc-lbl">axes</span>
<select class="cc-sync" id="cc-sync">
<option value="independent" selected>independent</option>
<option value="locked">locked</option>
</select>
</label>
</div>
<div class="compare-grid">
<article class="compare-panel" data-slot="a">
<header class="compare-panel-head">
<span class="panel-tag" data-role="label">A</span>
<span class="panel-embedder" data-role="embedder">&hellip;</span>
<span class="panel-sep">&middot;</span>
<span class="panel-generator" data-role="generator">&hellip;</span>
<a class="panel-stem" data-role="stem-link"
href="/api/runs/{{ stem_a }}/frames.json" target="_blank">&#x2197;</a>
<span class="panel-params" data-role="params">loading&hellip;</span>
</header>
<div class="compare-canvas" data-role="canvas">
<div class="compare-status" data-role="status">loading&hellip;</div>
</div>
</article>
<article class="compare-panel" data-slot="b">
<header class="compare-panel-head">
<span class="panel-tag" data-role="label">B</span>
<span class="panel-embedder" data-role="embedder">&hellip;</span>
<span class="panel-sep">&middot;</span>
<span class="panel-generator" data-role="generator">&hellip;</span>
<a class="panel-stem" data-role="stem-link"
href="/api/runs/{{ stem_b }}/frames.json" target="_blank">&#x2197;</a>
<span class="panel-params" data-role="params">loading&hellip;</span>
</header>
<div class="compare-canvas" data-role="canvas">
<div class="compare-status" data-role="status">loading&hellip;</div>
</div>
</article>
</div>
</section>
<script src="/static/theme.js?v=11"></script>
<script type="module" src="/static/compare.js?v=11"></script>
</body>
</html>

14
app/web/templates/index.html
View File

@ -4,7 +4,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title>embedding notebook</title>
<link rel="stylesheet" href="/static/style.css?v=27" />
<link rel="stylesheet" href="/static/style.css?v=17" />
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
<script type="importmap">
{
@ -260,11 +260,11 @@
hx-trigger="change" />
<label for="red-{{ loop.index }}">
<span class="mark">{{ "%02d"|format(loop.index) }}</span>
<span class="text">
<span>
<span class="name">{{ r.label }}</span>
<span class="blurb">{{ r.blurb }}</span>
</span>
<span class="pkg">{{ r.key.replace('.', '.<wbr>')|safe }}</span>
<span class="pkg">{{ r.key }}</span>
</label>
</li>
{% endfor %}
@ -301,13 +301,6 @@
</span>
</div>
<div class="compare-bar">
<button type="button" id="compare-btn" disabled>
compare selected <span id="compare-count">(0/2)</span>
</button>
<span class="compare-hint muted">pick two embeddings &rarr; side-by-side animation in a new tab</span>
</div>
<div
id="runs-slot"
hx-get="/runs"
@ -407,7 +400,6 @@
<script src="/static/theme.js?v=11"></script>
<script type="module" src="/static/dataset-picker.js?v=11"></script>
<script type="module" src="/static/metrics.js?v=11"></script>
<script src="/static/compare-select.js?v=1"></script>
<script>
// Anchor-links alone don't expand <details>; force it.
document.querySelector('a[href="#metrics"]')?.addEventListener('click', () => {

35
flows/embedding_flow.py
View File

@ -20,19 +20,11 @@ os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
os.environ.setdefault("NUMBA_NUM_THREADS", "1")
from datetime import timedelta
import hashlib
import json
import math
from pathlib import Path
from typing import Any, Dict, List, Optional
def _embed_args_hash(ea: Optional[Dict[str, Any]]) -> str:
"""8-hex digest of embed_args (keys sorted) — output stem includes this
so runs differing only in embed_args get distinct files."""
s = json.dumps(ea or {}, sort_keys=True, default=str)
return hashlib.sha1(s.encode()).hexdigest()[:8]
from prefect import flow, task
from prefect.artifacts import create_markdown_artifact, create_table_artifact
from prefect.cache_policies import INPUTS, NO_CACHE
@ -287,12 +279,10 @@ def embedding_flow(
output_ref: str = (
f"{output_dir.strip('/')}/{_generator}_Reference_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}.html"
)
_args_tag = _embed_args_hash(embed_args)
output_embed: str = (
f"{output_dir.strip('/')}/{_generator}_{embedder.split('.')[-1]}_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}_{_args_tag}.html"
f"{output_dir.strip('/')}/{_generator}_{embedder.split('.')[-1]}_N{num_points}_T{num_timesteps}_J{jitter_scale}_s{seed}.html"
)
output_metrics: str = output_embed[:-5] + ".metrics.json"
output_frames: str = output_embed[:-5] + ".frames.json"
title_ref = f"Reference: {_generator}, N={num_points} with {jitter_scale} noise"
title_embed = f"Embedding: {embedder.split('.')[-1]} on {_generator}, N={num_points} with {jitter_scale} noise"
@ -388,28 +378,7 @@ def embedding_flow(
k=10,
)
emb_path_result = emb_path.result()
metrics_path_result = metrics_path.result()
# Emit a frames.json sidecar so the compare page doesn't have to parse
# the 5 MB plotly HTML on every first request. Non-critical — the server
# falls back to HTML parsing when the sidecar is absent.
try:
import sys as _sys
_root = str(Path(__file__).resolve().parent.parent)
if _root not in _sys.path:
_sys.path.insert(0, _root)
from app.web.plotly_parse import parse_plotly_run
frames = parse_plotly_run(emb_path_result)
Path(output_frames).write_text(
json.dumps(frames, separators=(",", ":")), encoding="utf-8"
)
except Exception as _sidecar_err:
import traceback as _tb
print(f"[frames-sidecar] skipped: {_sidecar_err}")
_tb.print_exc()
return (ref_path.result(), emb_path_result, metrics_path_result)
return (ref_path.result(), emb_path.result(), metrics_path.result())
if __name__ == "__main__":