<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Language Learning - Live Stream</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: 'Monaco', 'Consolas', monospace;
background: #0a0a0f;
color: #e0e0e0;
overflow: hidden;
}
#container {
display: grid;
grid-template-columns: 1fr 280px;
grid-template-rows: 1fr 200px;
height: 100vh;
gap: 1px;
background: #222;
}
#graph {
background: #0a0a0f;
position: relative;
overflow: hidden;
}
#graph canvas { display: block; }
#sidebar {
background: #12121a;
padding: 16px;
overflow-y: auto;
grid-row: 1 / 3;
grid-column: 2;
border-left: 1px solid #333;
}
#timeline {
background: #0d0d14;
grid-column: 1;
position: relative;
overflow: hidden;
}
#timeline canvas { display: block; }
h1 { font-size: 13px; margin-bottom: 12px; color: #888; letter-spacing: 1px; }
h2 { font-size: 11px; margin: 14px 0 8px; color: #555; text-transform: uppercase; letter-spacing: 1px; }
.stat {
display: flex; justify-content: space-between;
padding: 3px 0; font-size: 11px;
}
.stat-value { color: #4ecdc4; font-weight: bold; }
.stat-value.surprise { color: #ff6b6b; }
.stat-value.learn { color: #ffd93d; }
.stat-value.cluster { color: #a78bfa; }
#event-log {
margin-top: 12px;
font-size: 10px;
max-height: 200px;
overflow-y: auto;
border-top: 1px solid #333;
padding-top: 8px;
}
.log-entry { padding: 2px 0; color: #666; }
.log-entry.merge { color: #a78bfa; }
.log-entry.mi { color: #4ecdc4; }
.log-entry.calibrate { color: #ffd93d; }
.log-entry.promote { color: #ff6b6b; font-weight: bold; }
#status-bar {
position: absolute; bottom: 0; left: 0; right: 0;
height: 24px; background: #1a1a2e;
display: flex; align-items: center;
padding: 0 12px; font-size: 10px; color: #555;
border-top: 1px solid #333;
}
#status-bar span { margin-right: 20px; }
#status-bar .live { color: #4ecdc4; }
#status-bar .waiting { color: #666; }
.cluster-list {
font-size: 10px;
max-height: 150px;
overflow-y: auto;
}
.cluster-item {
padding: 3px 0;
border-bottom: 1px solid #1a1a2e;
}
.cluster-id { color: #a78bfa; margin-right: 6px; }
.cluster-words { color: #888; }
</style>
</head>
<body>
<div id="container">
<div id="graph">
<canvas id="graphCanvas"></canvas>
<div id="status-bar">
<span id="connStatus" class="waiting">WAITING</span>
<span id="fps">0 fps</span>
<span id="eventRate">0 events/s</span>
</div>
</div>
<div id="sidebar">
<h1>LIVE LEARNING</h1>
<h2>Progress</h2>
<div class="stat"><span>Sentences</span><span class="stat-value" id="statSent">0</span></div>
<div class="stat"><span>Learned</span><span class="stat-value learn" id="statLearns">0</span></div>
<div class="stat"><span>Learn rate</span><span class="stat-value learn" id="statLearnRate">0%</span></div>
<div class="stat"><span>Level</span><span class="stat-value" id="statLevel">0</span></div>
<h2>Surprise</h2>
<div class="stat"><span>Current</span><span class="stat-value surprise" id="statSurprise">0</span></div>
<div class="stat"><span>Threshold</span><span class="stat-value" id="statThreshold">0</span></div>
<div class="stat"><span>Baseline MI</span><span class="stat-value" id="statBaseline">0</span></div>
<h2>Clustering</h2>
<div class="stat"><span>Word classes</span><span class="stat-value cluster" id="statClasses">0</span></div>
<div class="stat"><span>Dirty pairs</span><span class="stat-value" id="statDirty">0</span></div>
<div class="stat"><span>MI cache</span><span class="stat-value" id="statMICache">0</span></div>
<h2>Clusters</h2>
<div id="clusterList" class="cluster-list"></div>
<h2>Events</h2>
<div id="event-log"></div>
</div>
<div id="timeline">
<canvas id="timelineCanvas"></canvas>
</div>
</div>
<script>
// ═══════════════════════════════════════════════════════════════
// STATE
// ═══════════════════════════════════════════════════════════════
const state = {
sentences: 0,
learns: 0,
level: 0,
surprise: 0,
threshold: 0,
baseline: 0,
classes: 0,
dirty: 0,
miCache: 0,
running: false,
// Time series (for timeline chart)
surpriseHistory: [], // [{i, sur}]
learnHistory: [], // [{i, rate}] rolling learn rate
classHistory: [], // [{i, n}]
// Graph nodes (words involved in merges)
nodes: new Map(), // word -> {x, y, vx, vy, cluster, size}
edges: [], // [{from, to, sim}]
clusters: new Map(), // clusterId -> [words]
// Event log
events: [],
linesRead: 0,
eventsPerSec: 0,
lastEventCount: 0,
lastEventTime: Date.now(),
};
// ═══════════════════════════════════════════════════════════════
// NDJSON POLLING
// ═══════════════════════════════════════════════════════════════
let pollInterval = null;
const POLL_MS = 100; // 10Hz polling
async function pollStream() {
try {
const resp = await fetch('stream.jsonl?t=' + Date.now());
if (!resp.ok) {
document.getElementById('connStatus').textContent = 'NO STREAM';
document.getElementById('connStatus').className = 'waiting';
return;
}
const text = await resp.text();
const lines = text.split('\n').filter(l => l.trim());
if (lines.length > state.linesRead) {
document.getElementById('connStatus').textContent = 'LIVE';
document.getElementById('connStatus').className = 'live';
state.running = true;
// Process new lines only
const newLines = lines.slice(state.linesRead);
for (const line of newLines) {
try {
const event = JSON.parse(line);
processEvent(event);
} catch (e) { /* skip malformed */ }
}
state.linesRead = lines.length;
}
} catch (e) {
document.getElementById('connStatus').textContent = 'WAITING';
document.getElementById('connStatus').className = 'waiting';
}
}
// ═══════════════════════════════════════════════════════════════
// EVENT PROCESSING
// ═══════════════════════════════════════════════════════════════
function processEvent(ev) {
switch (ev.t) {
case 'start':
state.threshold = ev.threshold || 0;
state.baseline = ev.baseline || 0;
state.level = ev.level || 0;
logEvent('start', `Starting: ${ev.total} sentences, L${ev.level}`);
break;
case 's': // sentence
state.sentences = ev.i;
state.surprise = ev.sur;
if (ev.learn) state.learns++;
// Record time series
state.surpriseHistory.push({ i: ev.i, sur: ev.sur });
if (state.surpriseHistory.length > 2000) state.surpriseHistory.shift();
// Rolling learn rate (window of 100)
const recentLearns = state.surpriseHistory.slice(-100).filter(s => s.sur > state.threshold).length;
state.learnHistory.push({ i: ev.i, rate: recentLearns / 100 });
if (state.learnHistory.length > 2000) state.learnHistory.shift();
break;
case 'merge':
addMergeToGraph(ev.w1, ev.w2, ev.sim, ev.c);
logEvent('merge', `C${ev.c}: "${ev.w1}" + "${ev.w2}" (${ev.sim.toFixed(3)})`);
break;
case 'cluster':
state.classes = ev.classes;
state.classHistory.push({ i: state.sentences, n: ev.classes });
if (state.classHistory.length > 2000) state.classHistory.shift();
logEvent('cluster', `Clustering: ${ev.merges} merges, ${ev.classes} classes (iter ${ev.iter})`);
break;
case 'mi':
state.baseline = ev.base;
state.miCache = ev.cache;
logEvent('mi', `MI recompute: ${ev.dirty} dirty, baseline=${ev.base.toFixed(3)}`);
break;
case 'calibrate':
state.threshold = ev.threshold;
logEvent('calibrate', `Threshold recalibrated: ${ev.threshold.toFixed(3)}`);
break;
case 'promote':
state.level = ev.to;
logEvent('promote', `PROMOTED: Level ${ev.from} -> Level ${ev.to}`);
break;
case 'status':
state.sentences = ev.sent;
state.learns = ev.learns;
state.classes = ev.classes;
state.dirty = ev.dirty;
state.level = ev.level;
state.baseline = ev.base;
break;
case 'end':
state.running = false;
document.getElementById('connStatus').textContent = 'DONE';
logEvent('end', `Complete: ${ev.sent} sentences, ${ev.classes} classes`);
break;
}
}
function addMergeToGraph(w1, w2, sim, clusterId) {
const cx = graphCanvas.width / 2;
const cy = graphCanvas.height / 2;
// Add nodes if new
if (!state.nodes.has(w1)) {
state.nodes.set(w1, {
x: cx + (Math.random() - 0.5) * 400,
y: cy + (Math.random() - 0.5) * 400,
vx: 0, vy: 0,
cluster: clusterId,
size: 4,
label: w1
});
}
if (!state.nodes.has(w2)) {
state.nodes.set(w2, {
x: cx + (Math.random() - 0.5) * 400,
y: cy + (Math.random() - 0.5) * 400,
vx: 0, vy: 0,
cluster: clusterId,
size: 4,
label: w2
});
}
// Update cluster assignment
state.nodes.get(w1).cluster = clusterId;
state.nodes.get(w2).cluster = clusterId;
state.nodes.get(w1).size = Math.min(12, state.nodes.get(w1).size + 1);
state.nodes.get(w2).size = Math.min(12, state.nodes.get(w2).size + 1);
// Add edge
state.edges.push({ from: w1, to: w2, sim, cluster: clusterId });
// Track cluster membership
if (!state.clusters.has(clusterId)) state.clusters.set(clusterId, []);
const cl = state.clusters.get(clusterId);
if (!cl.includes(w1)) cl.push(w1);
if (!cl.includes(w2)) cl.push(w2);
}
function logEvent(type, msg) {
state.events.push({ type, msg, time: Date.now() });
if (state.events.length > 100) state.events.shift();
}
// ═══════════════════════════════════════════════════════════════
// RENDERING (30fps)
// ═══════════════════════════════════════════════════════════════
const graphCanvas = document.getElementById('graphCanvas');
const graphCtx = graphCanvas.getContext('2d');
const timelineCanvas = document.getElementById('timelineCanvas');
const timelineCtx = timelineCanvas.getContext('2d');
const CLUSTER_COLORS = [
'#4ecdc4', '#ff6b6b', '#a78bfa', '#ffd93d', '#6bcb77',
'#ff9a3c', '#4ea8de', '#f472b6', '#34d399', '#fb923c',
'#818cf8', '#f87171', '#38bdf8', '#a3e635', '#e879f9',
'#22d3ee', '#facc15', '#fb7185', '#2dd4bf', '#c084fc'
];
function clusterColor(id) {
if (id < 0) return '#444';
return CLUSTER_COLORS[id % CLUSTER_COLORS.length];
}
let lastFrame = 0;
let frameCount = 0;
let fpsDisplay = 0;
function render(timestamp) {
requestAnimationFrame(render);
// FPS counting
frameCount++;
if (timestamp - lastFrame >= 1000) {
fpsDisplay = frameCount;
frameCount = 0;
lastFrame = timestamp;
// Events per second
const now = Date.now();
const dt = (now - state.lastEventTime) / 1000;
state.eventsPerSec = Math.round((state.linesRead - state.lastEventCount) / dt);
state.lastEventCount = state.linesRead;
state.lastEventTime = now;
document.getElementById('fps').textContent = fpsDisplay + ' fps';
document.getElementById('eventRate').textContent = state.eventsPerSec + ' events/s';
}
resizeCanvases();
renderGraph();
renderTimeline();
renderSidebar();
}
function resizeCanvases() {
const gp = graphCanvas.parentElement;
if (graphCanvas.width !== gp.clientWidth || graphCanvas.height !== gp.clientHeight) {
graphCanvas.width = gp.clientWidth;
graphCanvas.height = gp.clientHeight;
}
const tp = timelineCanvas.parentElement;
if (timelineCanvas.width !== tp.clientWidth || timelineCanvas.height !== tp.clientHeight) {
timelineCanvas.width = tp.clientWidth;
timelineCanvas.height = tp.clientHeight;
}
}
// ═══════════════════════════════════════════════════════════════
// FORCE-DIRECTED GRAPH
// ═══════════════════════════════════════════════════════════════
function renderGraph() {
const ctx = graphCtx;
const W = graphCanvas.width;
const H = graphCanvas.height;
ctx.fillStyle = '#0a0a0f';
ctx.fillRect(0, 0, W, H);
if (state.nodes.size === 0) {
ctx.fillStyle = '#333';
ctx.font = '14px Monaco, Consolas, monospace';
ctx.textAlign = 'center';
ctx.fillText('Waiting for merge events...', W/2, H/2);
return;
}
// Simple force simulation step
simulateForces();
// Draw edges
ctx.lineWidth = 1;
for (const edge of state.edges) {
const a = state.nodes.get(edge.from);
const b = state.nodes.get(edge.to);
if (!a || !b) continue;
ctx.strokeStyle = clusterColor(edge.cluster) + '40';
ctx.beginPath();
ctx.moveTo(a.x, a.y);
ctx.lineTo(b.x, b.y);
ctx.stroke();
}
// Draw nodes
for (const [word, node] of state.nodes) {
ctx.fillStyle = clusterColor(node.cluster);
ctx.beginPath();
ctx.arc(node.x, node.y, node.size, 0, Math.PI * 2);
ctx.fill();
// Label
ctx.fillStyle = '#ccc';
ctx.font = '10px Monaco, Consolas, monospace';
ctx.textAlign = 'center';
ctx.fillText(word, node.x, node.y - node.size - 4);
}
}
function simulateForces() {
const nodes = Array.from(state.nodes.values());
const cx = graphCanvas.width / 2;
const cy = graphCanvas.height / 2;
const dt = 0.3;
// Repulsion between all nodes
for (let i = 0; i < nodes.length; i++) {
for (let j = i + 1; j < nodes.length; j++) {
const a = nodes[i], b = nodes[j];
let dx = b.x - a.x, dy = b.y - a.y;
let dist = Math.sqrt(dx * dx + dy * dy) || 1;
let force = 500 / (dist * dist);
let fx = dx / dist * force, fy = dy / dist * force;
a.vx -= fx; a.vy -= fy;
b.vx += fx; b.vy += fy;
}
}
// Attraction along edges
for (const edge of state.edges) {
const a = state.nodes.get(edge.from);
const b = state.nodes.get(edge.to);
if (!a || !b) continue;
let dx = b.x - a.x, dy = b.y - a.y;
let dist = Math.sqrt(dx * dx + dy * dy) || 1;
let force = (dist - 80) * 0.01 * edge.sim;
let fx = dx / dist * force, fy = dy / dist * force;
a.vx += fx; a.vy += fy;
b.vx -= fx; b.vy -= fy;
}
// Center gravity
for (const node of nodes) {
node.vx += (cx - node.x) * 0.001;
node.vy += (cy - node.y) * 0.001;
}
// Apply velocity with damping
for (const node of nodes) {
node.vx *= 0.9;
node.vy *= 0.9;
node.x += node.vx * dt;
node.y += node.vy * dt;
// Keep in bounds
node.x = Math.max(40, Math.min(graphCanvas.width - 40, node.x));
node.y = Math.max(40, Math.min(graphCanvas.height - 60, node.y));
}
}
// ═══════════════════════════════════════════════════════════════
// TIMELINE CHART
// ═══════════════════════════════════════════════════════════════
function renderTimeline() {
const ctx = timelineCtx;
const W = timelineCanvas.width;
const H = timelineCanvas.height;
ctx.fillStyle = '#0d0d14';
ctx.fillRect(0, 0, W, H);
if (state.surpriseHistory.length < 2) return;
const padding = { top: 20, bottom: 30, left: 50, right: 20 };
const plotW = W - padding.left - padding.right;
const plotH = H - padding.top - padding.bottom;
// X axis: sentence index
const data = state.surpriseHistory;
const minI = data[0].i;
const maxI = data[data.length - 1].i;
const rangeI = maxI - minI || 1;
// Y axis: surprise (auto-scale)
let maxSur = 0;
for (const d of data) maxSur = Math.max(maxSur, d.sur);
maxSur = Math.max(maxSur, state.threshold * 1.2) || 10;
function toX(i) { return padding.left + ((i - minI) / rangeI) * plotW; }
function toY(v) { return padding.top + plotH - (v / maxSur) * plotH; }
// Grid
ctx.strokeStyle = '#1a1a2e';
ctx.lineWidth = 1;
for (let v = 0; v <= maxSur; v += Math.max(1, Math.floor(maxSur / 4))) {
const y = toY(v);
ctx.beginPath(); ctx.moveTo(padding.left, y); ctx.lineTo(W - padding.right, y); ctx.stroke();
ctx.fillStyle = '#444';
ctx.font = '9px Monaco, monospace';
ctx.textAlign = 'right';
ctx.fillText(v.toFixed(1), padding.left - 5, y + 3);
}
// Threshold line
ctx.strokeStyle = '#ffd93d50';
ctx.setLineDash([4, 4]);
ctx.beginPath();
ctx.moveTo(padding.left, toY(state.threshold));
ctx.lineTo(W - padding.right, toY(state.threshold));
ctx.stroke();
ctx.setLineDash([]);
ctx.fillStyle = '#ffd93d80';
ctx.font = '9px Monaco, monospace';
ctx.textAlign = 'left';
ctx.fillText('threshold', W - padding.right - 60, toY(state.threshold) - 4);
// Surprise line
ctx.strokeStyle = '#ff6b6b';
ctx.lineWidth = 1.5;
ctx.beginPath();
for (let k = 0; k < data.length; k++) {
const x = toX(data[k].i);
const y = toY(Math.min(data[k].sur, maxSur));
if (k === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.stroke();
// Learn rate line (if available)
if (state.learnHistory.length > 1) {
ctx.strokeStyle = '#4ecdc480';
ctx.lineWidth = 1;
ctx.beginPath();
for (let k = 0; k < state.learnHistory.length; k++) {
const d = state.learnHistory[k];
const x = toX(d.i);
const y = padding.top + plotH - d.rate * plotH;
if (k === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.stroke();
}
// Class count (right axis)
if (state.classHistory.length > 0) {
const maxC = Math.max(...state.classHistory.map(d => d.n)) || 1;
ctx.strokeStyle = '#a78bfa80';
ctx.lineWidth = 2;
ctx.beginPath();
for (let k = 0; k < state.classHistory.length; k++) {
const d = state.classHistory[k];
const x = toX(d.i);
const y = padding.top + plotH - (d.n / maxC) * plotH;
if (k === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
}
ctx.stroke();
}
// Labels
ctx.fillStyle = '#666';
ctx.font = '10px Monaco, monospace';
ctx.textAlign = 'center';
ctx.fillText(`Sentence ${minI} - ${maxI}`, W / 2, H - 5);
// Legend
ctx.textAlign = 'left';
const lx = padding.left + 10;
ctx.fillStyle = '#ff6b6b'; ctx.fillText('surprise', lx, padding.top - 5);
ctx.fillStyle = '#ffd93d80'; ctx.fillText('threshold', lx + 70, padding.top - 5);
ctx.fillStyle = '#4ecdc480'; ctx.fillText('learn rate', lx + 150, padding.top - 5);
ctx.fillStyle = '#a78bfa80'; ctx.fillText('classes', lx + 230, padding.top - 5);
}
// ═══════════════════════════════════════════════════════════════
// SIDEBAR UPDATES
// ═══════════════════════════════════════════════════════════════
function renderSidebar() {
document.getElementById('statSent').textContent = state.sentences.toLocaleString();
document.getElementById('statLearns').textContent = state.learns.toLocaleString();
document.getElementById('statLearnRate').textContent =
state.sentences > 0 ? (100 * state.learns / state.sentences).toFixed(1) + '%' : '0%';
document.getElementById('statLevel').textContent = state.level;
document.getElementById('statSurprise').textContent =
typeof state.surprise === 'number' ? state.surprise.toFixed(3) : '0';
document.getElementById('statThreshold').textContent =
typeof state.threshold === 'number' ? state.threshold.toFixed(3) : '0';
document.getElementById('statBaseline').textContent =
typeof state.baseline === 'number' ? state.baseline.toFixed(3) : '0';
document.getElementById('statClasses').textContent = state.classes;
document.getElementById('statDirty').textContent =
typeof state.dirty === 'number' ? state.dirty.toLocaleString() : '0';
document.getElementById('statMICache').textContent =
typeof state.miCache === 'number' ? state.miCache.toLocaleString() : '0';
// Cluster list
const clEl = document.getElementById('clusterList');
if (state.clusters.size > 0) {
let html = '';
for (const [id, words] of state.clusters) {
html += `<div class="cluster-item"><span class="cluster-id" style="color:${clusterColor(id)}">C${id}</span>`;
html += `<span class="cluster-words">${words.join(', ')}</span></div>`;
}
clEl.innerHTML = html;
}
// Event log
const logEl = document.getElementById('event-log');
const recent = state.events.slice(-20);
let logHtml = '';
for (const ev of recent) {
logHtml += `<div class="log-entry ${ev.type}">${ev.msg}</div>`;
}
logEl.innerHTML = logHtml;
logEl.scrollTop = logEl.scrollHeight;
}
// ═══════════════════════════════════════════════════════════════
// INIT
// ═══════════════════════════════════════════════════════════════
pollInterval = setInterval(pollStream, POLL_MS);
pollStream(); // Initial poll
requestAnimationFrame(render);
</script>
</body>
</html>