OverviewFilesCommitsIssuesDocsReleasesPeers

hypercore-kb / opencog-kb / language-learning / mvp / viz / archive / live.html

661 lines · html ref:
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Language Learning - Live Stream</title>
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }
    body {
      font-family: 'Monaco', 'Consolas', monospace;
      background: #0a0a0f;
      color: #e0e0e0;
      overflow: hidden;
    }
    #container {
      display: grid;
      grid-template-columns: 1fr 280px;
      grid-template-rows: 1fr 200px;
      height: 100vh;
      gap: 1px;
      background: #222;
    }
    #graph {
      background: #0a0a0f;
      position: relative;
      overflow: hidden;
    }
    #graph canvas { display: block; }
    #sidebar {
      background: #12121a;
      padding: 16px;
      overflow-y: auto;
      grid-row: 1 / 3;
      grid-column: 2;
      border-left: 1px solid #333;
    }
    #timeline {
      background: #0d0d14;
      grid-column: 1;
      position: relative;
      overflow: hidden;
    }
    #timeline canvas { display: block; }

    h1 { font-size: 13px; margin-bottom: 12px; color: #888; letter-spacing: 1px; }
    h2 { font-size: 11px; margin: 14px 0 8px; color: #555; text-transform: uppercase; letter-spacing: 1px; }
    .stat {
      display: flex; justify-content: space-between;
      padding: 3px 0; font-size: 11px;
    }
    .stat-value { color: #4ecdc4; font-weight: bold; }
    .stat-value.surprise { color: #ff6b6b; }
    .stat-value.learn { color: #ffd93d; }
    .stat-value.cluster { color: #a78bfa; }

    #event-log {
      margin-top: 12px;
      font-size: 10px;
      max-height: 200px;
      overflow-y: auto;
      border-top: 1px solid #333;
      padding-top: 8px;
    }
    .log-entry { padding: 2px 0; color: #666; }
    .log-entry.merge { color: #a78bfa; }
    .log-entry.mi { color: #4ecdc4; }
    .log-entry.calibrate { color: #ffd93d; }
    .log-entry.promote { color: #ff6b6b; font-weight: bold; }

    #status-bar {
      position: absolute; bottom: 0; left: 0; right: 0;
      height: 24px; background: #1a1a2e;
      display: flex; align-items: center;
      padding: 0 12px; font-size: 10px; color: #555;
      border-top: 1px solid #333;
    }
    #status-bar span { margin-right: 20px; }
    #status-bar .live { color: #4ecdc4; }
    #status-bar .waiting { color: #666; }

    .cluster-list {
      font-size: 10px;
      max-height: 150px;
      overflow-y: auto;
    }
    .cluster-item {
      padding: 3px 0;
      border-bottom: 1px solid #1a1a2e;
    }
    .cluster-id { color: #a78bfa; margin-right: 6px; }
    .cluster-words { color: #888; }
  </style>
</head>
<body>
  <div id="container">
    <div id="graph">
      <canvas id="graphCanvas"></canvas>
      <div id="status-bar">
        <span id="connStatus" class="waiting">WAITING</span>
        <span id="fps">0 fps</span>
        <span id="eventRate">0 events/s</span>
      </div>
    </div>
    <div id="sidebar">
      <h1>LIVE LEARNING</h1>

      <h2>Progress</h2>
      <div class="stat"><span>Sentences</span><span class="stat-value" id="statSent">0</span></div>
      <div class="stat"><span>Learned</span><span class="stat-value learn" id="statLearns">0</span></div>
      <div class="stat"><span>Learn rate</span><span class="stat-value learn" id="statLearnRate">0%</span></div>
      <div class="stat"><span>Level</span><span class="stat-value" id="statLevel">0</span></div>

      <h2>Surprise</h2>
      <div class="stat"><span>Current</span><span class="stat-value surprise" id="statSurprise">0</span></div>
      <div class="stat"><span>Threshold</span><span class="stat-value" id="statThreshold">0</span></div>
      <div class="stat"><span>Baseline MI</span><span class="stat-value" id="statBaseline">0</span></div>

      <h2>Clustering</h2>
      <div class="stat"><span>Word classes</span><span class="stat-value cluster" id="statClasses">0</span></div>
      <div class="stat"><span>Dirty pairs</span><span class="stat-value" id="statDirty">0</span></div>
      <div class="stat"><span>MI cache</span><span class="stat-value" id="statMICache">0</span></div>

      <h2>Clusters</h2>
      <div id="clusterList" class="cluster-list"></div>

      <h2>Events</h2>
      <div id="event-log"></div>
    </div>
    <div id="timeline">
      <canvas id="timelineCanvas"></canvas>
    </div>
  </div>

  <script>
  // ═══════════════════════════════════════════════════════════════
  // STATE
  // ═══════════════════════════════════════════════════════════════

  const state = {
    sentences: 0,
    learns: 0,
    level: 0,
    surprise: 0,
    threshold: 0,
    baseline: 0,
    classes: 0,
    dirty: 0,
    miCache: 0,
    running: false,

    // Time series (for timeline chart)
    surpriseHistory: [],    // [{i, sur}]
    learnHistory: [],       // [{i, rate}] rolling learn rate
    classHistory: [],       // [{i, n}]

    // Graph nodes (words involved in merges)
    nodes: new Map(),       // word -> {x, y, vx, vy, cluster, size}
    edges: [],              // [{from, to, sim}]
    clusters: new Map(),    // clusterId -> [words]

    // Event log
    events: [],
    linesRead: 0,
    eventsPerSec: 0,
    lastEventCount: 0,
    lastEventTime: Date.now(),
  };

  // ═══════════════════════════════════════════════════════════════
  // NDJSON POLLING
  // ═══════════════════════════════════════════════════════════════

  let pollInterval = null;
  const POLL_MS = 100; // 10Hz polling

  async function pollStream() {
    try {
      const resp = await fetch('stream.jsonl?t=' + Date.now());
      if (!resp.ok) {
        document.getElementById('connStatus').textContent = 'NO STREAM';
        document.getElementById('connStatus').className = 'waiting';
        return;
      }
      const text = await resp.text();
      const lines = text.split('\n').filter(l => l.trim());

      if (lines.length > state.linesRead) {
        document.getElementById('connStatus').textContent = 'LIVE';
        document.getElementById('connStatus').className = 'live';
        state.running = true;

        // Process new lines only
        const newLines = lines.slice(state.linesRead);
        for (const line of newLines) {
          try {
            const event = JSON.parse(line);
            processEvent(event);
          } catch (e) { /* skip malformed */ }
        }
        state.linesRead = lines.length;
      }
    } catch (e) {
      document.getElementById('connStatus').textContent = 'WAITING';
      document.getElementById('connStatus').className = 'waiting';
    }
  }

  // ═══════════════════════════════════════════════════════════════
  // EVENT PROCESSING
  // ═══════════════════════════════════════════════════════════════

  function processEvent(ev) {
    switch (ev.t) {
      case 'start':
        state.threshold = ev.threshold || 0;
        state.baseline = ev.baseline || 0;
        state.level = ev.level || 0;
        logEvent('start', `Starting: ${ev.total} sentences, L${ev.level}`);
        break;

      case 's': // sentence
        state.sentences = ev.i;
        state.surprise = ev.sur;
        if (ev.learn) state.learns++;
        // Record time series
        state.surpriseHistory.push({ i: ev.i, sur: ev.sur });
        if (state.surpriseHistory.length > 2000) state.surpriseHistory.shift();
        // Rolling learn rate (window of 100)
        const recentLearns = state.surpriseHistory.slice(-100).filter(s => s.sur > state.threshold).length;
        state.learnHistory.push({ i: ev.i, rate: recentLearns / 100 });
        if (state.learnHistory.length > 2000) state.learnHistory.shift();
        break;

      case 'merge':
        addMergeToGraph(ev.w1, ev.w2, ev.sim, ev.c);
        logEvent('merge', `C${ev.c}: "${ev.w1}" + "${ev.w2}" (${ev.sim.toFixed(3)})`);
        break;

      case 'cluster':
        state.classes = ev.classes;
        state.classHistory.push({ i: state.sentences, n: ev.classes });
        if (state.classHistory.length > 2000) state.classHistory.shift();
        logEvent('cluster', `Clustering: ${ev.merges} merges, ${ev.classes} classes (iter ${ev.iter})`);
        break;

      case 'mi':
        state.baseline = ev.base;
        state.miCache = ev.cache;
        logEvent('mi', `MI recompute: ${ev.dirty} dirty, baseline=${ev.base.toFixed(3)}`);
        break;

      case 'calibrate':
        state.threshold = ev.threshold;
        logEvent('calibrate', `Threshold recalibrated: ${ev.threshold.toFixed(3)}`);
        break;

      case 'promote':
        state.level = ev.to;
        logEvent('promote', `PROMOTED: Level ${ev.from} -> Level ${ev.to}`);
        break;

      case 'status':
        state.sentences = ev.sent;
        state.learns = ev.learns;
        state.classes = ev.classes;
        state.dirty = ev.dirty;
        state.level = ev.level;
        state.baseline = ev.base;
        break;

      case 'end':
        state.running = false;
        document.getElementById('connStatus').textContent = 'DONE';
        logEvent('end', `Complete: ${ev.sent} sentences, ${ev.classes} classes`);
        break;
    }
  }

  function addMergeToGraph(w1, w2, sim, clusterId) {
    const cx = graphCanvas.width / 2;
    const cy = graphCanvas.height / 2;

    // Add nodes if new
    if (!state.nodes.has(w1)) {
      state.nodes.set(w1, {
        x: cx + (Math.random() - 0.5) * 400,
        y: cy + (Math.random() - 0.5) * 400,
        vx: 0, vy: 0,
        cluster: clusterId,
        size: 4,
        label: w1
      });
    }
    if (!state.nodes.has(w2)) {
      state.nodes.set(w2, {
        x: cx + (Math.random() - 0.5) * 400,
        y: cy + (Math.random() - 0.5) * 400,
        vx: 0, vy: 0,
        cluster: clusterId,
        size: 4,
        label: w2
      });
    }

    // Update cluster assignment
    state.nodes.get(w1).cluster = clusterId;
    state.nodes.get(w2).cluster = clusterId;
    state.nodes.get(w1).size = Math.min(12, state.nodes.get(w1).size + 1);
    state.nodes.get(w2).size = Math.min(12, state.nodes.get(w2).size + 1);

    // Add edge
    state.edges.push({ from: w1, to: w2, sim, cluster: clusterId });

    // Track cluster membership
    if (!state.clusters.has(clusterId)) state.clusters.set(clusterId, []);
    const cl = state.clusters.get(clusterId);
    if (!cl.includes(w1)) cl.push(w1);
    if (!cl.includes(w2)) cl.push(w2);
  }

  function logEvent(type, msg) {
    state.events.push({ type, msg, time: Date.now() });
    if (state.events.length > 100) state.events.shift();
  }

  // ═══════════════════════════════════════════════════════════════
  // RENDERING (30fps)
  // ═══════════════════════════════════════════════════════════════

  const graphCanvas = document.getElementById('graphCanvas');
  const graphCtx = graphCanvas.getContext('2d');
  const timelineCanvas = document.getElementById('timelineCanvas');
  const timelineCtx = timelineCanvas.getContext('2d');

  const CLUSTER_COLORS = [
    '#4ecdc4', '#ff6b6b', '#a78bfa', '#ffd93d', '#6bcb77',
    '#ff9a3c', '#4ea8de', '#f472b6', '#34d399', '#fb923c',
    '#818cf8', '#f87171', '#38bdf8', '#a3e635', '#e879f9',
    '#22d3ee', '#facc15', '#fb7185', '#2dd4bf', '#c084fc'
  ];

  function clusterColor(id) {
    if (id < 0) return '#444';
    return CLUSTER_COLORS[id % CLUSTER_COLORS.length];
  }

  let lastFrame = 0;
  let frameCount = 0;
  let fpsDisplay = 0;

  function render(timestamp) {
    requestAnimationFrame(render);

    // FPS counting
    frameCount++;
    if (timestamp - lastFrame >= 1000) {
      fpsDisplay = frameCount;
      frameCount = 0;
      lastFrame = timestamp;

      // Events per second
      const now = Date.now();
      const dt = (now - state.lastEventTime) / 1000;
      state.eventsPerSec = Math.round((state.linesRead - state.lastEventCount) / dt);
      state.lastEventCount = state.linesRead;
      state.lastEventTime = now;

      document.getElementById('fps').textContent = fpsDisplay + ' fps';
      document.getElementById('eventRate').textContent = state.eventsPerSec + ' events/s';
    }

    resizeCanvases();
    renderGraph();
    renderTimeline();
    renderSidebar();
  }

  function resizeCanvases() {
    const gp = graphCanvas.parentElement;
    if (graphCanvas.width !== gp.clientWidth || graphCanvas.height !== gp.clientHeight) {
      graphCanvas.width = gp.clientWidth;
      graphCanvas.height = gp.clientHeight;
    }
    const tp = timelineCanvas.parentElement;
    if (timelineCanvas.width !== tp.clientWidth || timelineCanvas.height !== tp.clientHeight) {
      timelineCanvas.width = tp.clientWidth;
      timelineCanvas.height = tp.clientHeight;
    }
  }

  // ═══════════════════════════════════════════════════════════════
  // FORCE-DIRECTED GRAPH
  // ═══════════════════════════════════════════════════════════════

  function renderGraph() {
    const ctx = graphCtx;
    const W = graphCanvas.width;
    const H = graphCanvas.height;

    ctx.fillStyle = '#0a0a0f';
    ctx.fillRect(0, 0, W, H);

    if (state.nodes.size === 0) {
      ctx.fillStyle = '#333';
      ctx.font = '14px Monaco, Consolas, monospace';
      ctx.textAlign = 'center';
      ctx.fillText('Waiting for merge events...', W/2, H/2);
      return;
    }

    // Simple force simulation step
    simulateForces();

    // Draw edges
    ctx.lineWidth = 1;
    for (const edge of state.edges) {
      const a = state.nodes.get(edge.from);
      const b = state.nodes.get(edge.to);
      if (!a || !b) continue;
      ctx.strokeStyle = clusterColor(edge.cluster) + '40';
      ctx.beginPath();
      ctx.moveTo(a.x, a.y);
      ctx.lineTo(b.x, b.y);
      ctx.stroke();
    }

    // Draw nodes
    for (const [word, node] of state.nodes) {
      ctx.fillStyle = clusterColor(node.cluster);
      ctx.beginPath();
      ctx.arc(node.x, node.y, node.size, 0, Math.PI * 2);
      ctx.fill();

      // Label
      ctx.fillStyle = '#ccc';
      ctx.font = '10px Monaco, Consolas, monospace';
      ctx.textAlign = 'center';
      ctx.fillText(word, node.x, node.y - node.size - 4);
    }
  }

  function simulateForces() {
    const nodes = Array.from(state.nodes.values());
    const cx = graphCanvas.width / 2;
    const cy = graphCanvas.height / 2;
    const dt = 0.3;

    // Repulsion between all nodes
    for (let i = 0; i < nodes.length; i++) {
      for (let j = i + 1; j < nodes.length; j++) {
        const a = nodes[i], b = nodes[j];
        let dx = b.x - a.x, dy = b.y - a.y;
        let dist = Math.sqrt(dx * dx + dy * dy) || 1;
        let force = 500 / (dist * dist);
        let fx = dx / dist * force, fy = dy / dist * force;
        a.vx -= fx; a.vy -= fy;
        b.vx += fx; b.vy += fy;
      }
    }

    // Attraction along edges
    for (const edge of state.edges) {
      const a = state.nodes.get(edge.from);
      const b = state.nodes.get(edge.to);
      if (!a || !b) continue;
      let dx = b.x - a.x, dy = b.y - a.y;
      let dist = Math.sqrt(dx * dx + dy * dy) || 1;
      let force = (dist - 80) * 0.01 * edge.sim;
      let fx = dx / dist * force, fy = dy / dist * force;
      a.vx += fx; a.vy += fy;
      b.vx -= fx; b.vy -= fy;
    }

    // Center gravity
    for (const node of nodes) {
      node.vx += (cx - node.x) * 0.001;
      node.vy += (cy - node.y) * 0.001;
    }

    // Apply velocity with damping
    for (const node of nodes) {
      node.vx *= 0.9;
      node.vy *= 0.9;
      node.x += node.vx * dt;
      node.y += node.vy * dt;
      // Keep in bounds
      node.x = Math.max(40, Math.min(graphCanvas.width - 40, node.x));
      node.y = Math.max(40, Math.min(graphCanvas.height - 60, node.y));
    }
  }

  // ═══════════════════════════════════════════════════════════════
  // TIMELINE CHART
  // ═══════════════════════════════════════════════════════════════

  function renderTimeline() {
    const ctx = timelineCtx;
    const W = timelineCanvas.width;
    const H = timelineCanvas.height;

    ctx.fillStyle = '#0d0d14';
    ctx.fillRect(0, 0, W, H);

    if (state.surpriseHistory.length < 2) return;

    const padding = { top: 20, bottom: 30, left: 50, right: 20 };
    const plotW = W - padding.left - padding.right;
    const plotH = H - padding.top - padding.bottom;

    // X axis: sentence index
    const data = state.surpriseHistory;
    const minI = data[0].i;
    const maxI = data[data.length - 1].i;
    const rangeI = maxI - minI || 1;

    // Y axis: surprise (auto-scale)
    let maxSur = 0;
    for (const d of data) maxSur = Math.max(maxSur, d.sur);
    maxSur = Math.max(maxSur, state.threshold * 1.2) || 10;

    function toX(i) { return padding.left + ((i - minI) / rangeI) * plotW; }
    function toY(v) { return padding.top + plotH - (v / maxSur) * plotH; }

    // Grid
    ctx.strokeStyle = '#1a1a2e';
    ctx.lineWidth = 1;
    for (let v = 0; v <= maxSur; v += Math.max(1, Math.floor(maxSur / 4))) {
      const y = toY(v);
      ctx.beginPath(); ctx.moveTo(padding.left, y); ctx.lineTo(W - padding.right, y); ctx.stroke();
      ctx.fillStyle = '#444';
      ctx.font = '9px Monaco, monospace';
      ctx.textAlign = 'right';
      ctx.fillText(v.toFixed(1), padding.left - 5, y + 3);
    }

    // Threshold line
    ctx.strokeStyle = '#ffd93d50';
    ctx.setLineDash([4, 4]);
    ctx.beginPath();
    ctx.moveTo(padding.left, toY(state.threshold));
    ctx.lineTo(W - padding.right, toY(state.threshold));
    ctx.stroke();
    ctx.setLineDash([]);
    ctx.fillStyle = '#ffd93d80';
    ctx.font = '9px Monaco, monospace';
    ctx.textAlign = 'left';
    ctx.fillText('threshold', W - padding.right - 60, toY(state.threshold) - 4);

    // Surprise line
    ctx.strokeStyle = '#ff6b6b';
    ctx.lineWidth = 1.5;
    ctx.beginPath();
    for (let k = 0; k < data.length; k++) {
      const x = toX(data[k].i);
      const y = toY(Math.min(data[k].sur, maxSur));
      if (k === 0) ctx.moveTo(x, y);
      else ctx.lineTo(x, y);
    }
    ctx.stroke();

    // Learn rate line (if available)
    if (state.learnHistory.length > 1) {
      ctx.strokeStyle = '#4ecdc480';
      ctx.lineWidth = 1;
      ctx.beginPath();
      for (let k = 0; k < state.learnHistory.length; k++) {
        const d = state.learnHistory[k];
        const x = toX(d.i);
        const y = padding.top + plotH - d.rate * plotH;
        if (k === 0) ctx.moveTo(x, y);
        else ctx.lineTo(x, y);
      }
      ctx.stroke();
    }

    // Class count (right axis)
    if (state.classHistory.length > 0) {
      const maxC = Math.max(...state.classHistory.map(d => d.n)) || 1;
      ctx.strokeStyle = '#a78bfa80';
      ctx.lineWidth = 2;
      ctx.beginPath();
      for (let k = 0; k < state.classHistory.length; k++) {
        const d = state.classHistory[k];
        const x = toX(d.i);
        const y = padding.top + plotH - (d.n / maxC) * plotH;
        if (k === 0) ctx.moveTo(x, y);
        else ctx.lineTo(x, y);
      }
      ctx.stroke();
    }

    // Labels
    ctx.fillStyle = '#666';
    ctx.font = '10px Monaco, monospace';
    ctx.textAlign = 'center';
    ctx.fillText(`Sentence ${minI} - ${maxI}`, W / 2, H - 5);

    // Legend
    ctx.textAlign = 'left';
    const lx = padding.left + 10;
    ctx.fillStyle = '#ff6b6b'; ctx.fillText('surprise', lx, padding.top - 5);
    ctx.fillStyle = '#ffd93d80'; ctx.fillText('threshold', lx + 70, padding.top - 5);
    ctx.fillStyle = '#4ecdc480'; ctx.fillText('learn rate', lx + 150, padding.top - 5);
    ctx.fillStyle = '#a78bfa80'; ctx.fillText('classes', lx + 230, padding.top - 5);
  }

  // ═══════════════════════════════════════════════════════════════
  // SIDEBAR UPDATES
  // ═══════════════════════════════════════════════════════════════

  function renderSidebar() {
    document.getElementById('statSent').textContent = state.sentences.toLocaleString();
    document.getElementById('statLearns').textContent = state.learns.toLocaleString();
    document.getElementById('statLearnRate').textContent =
      state.sentences > 0 ? (100 * state.learns / state.sentences).toFixed(1) + '%' : '0%';
    document.getElementById('statLevel').textContent = state.level;
    document.getElementById('statSurprise').textContent =
      typeof state.surprise === 'number' ? state.surprise.toFixed(3) : '0';
    document.getElementById('statThreshold').textContent =
      typeof state.threshold === 'number' ? state.threshold.toFixed(3) : '0';
    document.getElementById('statBaseline').textContent =
      typeof state.baseline === 'number' ? state.baseline.toFixed(3) : '0';
    document.getElementById('statClasses').textContent = state.classes;
    document.getElementById('statDirty').textContent =
      typeof state.dirty === 'number' ? state.dirty.toLocaleString() : '0';
    document.getElementById('statMICache').textContent =
      typeof state.miCache === 'number' ? state.miCache.toLocaleString() : '0';

    // Cluster list
    const clEl = document.getElementById('clusterList');
    if (state.clusters.size > 0) {
      let html = '';
      for (const [id, words] of state.clusters) {
        html += `<div class="cluster-item"><span class="cluster-id" style="color:${clusterColor(id)}">C${id}</span>`;
        html += `<span class="cluster-words">${words.join(', ')}</span></div>`;
      }
      clEl.innerHTML = html;
    }

    // Event log
    const logEl = document.getElementById('event-log');
    const recent = state.events.slice(-20);
    let logHtml = '';
    for (const ev of recent) {
      logHtml += `<div class="log-entry ${ev.type}">${ev.msg}</div>`;
    }
    logEl.innerHTML = logHtml;
    logEl.scrollTop = logEl.scrollHeight;
  }

  // ═══════════════════════════════════════════════════════════════
  // INIT
  // ═══════════════════════════════════════════════════════════════

  pollInterval = setInterval(pollStream, POLL_MS);
  pollStream(); // Initial poll
  requestAnimationFrame(render);
  </script>
</body>
</html>