OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-proto-peer / lib / conflict-watch.js

127 lines · javascript ref:
// conflict-watch — make hypercore conflicts VISIBLE.
//
// WHY THIS EXISTS. When two copies of one core disagree, hypercore freezes the core, closes every session
// on it with "Two conflicting signatures exist for length N", and stops replicating it. The only trace it
// leaves anywhere is a `console.log` at hypercore/lib/core.js:685 that upstream literally labels
// `// tmp log so we can see these`. Nothing in this stack listens for the `conflict` event — not hpp, not
// atproto-room, not autobase 7.28.1 — so no diagnostic surface can report one. loser-club logged conflicts
// on core 4a6f1b30 for weeks while `diag` reported a perfectly healthy node, because a healthy-looking node
// is exactly what a conflict produces: the served view stays correct, one core just silently stops syncing.
//
// PER-PROCESS BY DESIGN, and this is not a limitation. `checkConflict` sets `header.frozen = true` and
// writes it through `setAuth`, but the `@core/auth` encoding has no `frozen` field
// (hypercore-storage/encoding/spec/hyperschema/index.js:440) — it is dropped on encode and never read back.
// So hypercore's own conflict state is in-memory for the life of the process, and a registry that outlived
// the process would claim a freeze the running daemon does not actually have. "Conflicts seen since boot"
// is the honest window, and it is the same window hypercore itself uses.
//
// WHY IT HOOKS THE INTERNAL CORE. The `conflict` event is emitted on a Core's monitors
// (core.js:916-919), and most cores in this daemon are opened by AUTOBASE, not by us — we never see those
// sessions, so there is nothing of ours to attach a listener to. `corestore.watch(fn)` fires for every core
// opened in the store (and its namespaces, which share one core tracker), which is the only hook that
// reaches all of them. It yields the internal Core, so registration goes through `addMonitor` rather than
// `.on()`. That is an internal API: `test/conflict-watch.test.js` drives a REAL conflict between two cores
// end to end, so if upstream ever changes how conflicts are emitted this fails loudly instead of quietly
// reporting zero forever — which would be worse than not having it at all.
import b4a from 'b4a'

const hex = (b) => (b ? b4a.toString(b, 'hex') : null)

// Bounded: a conflicting core that keeps being re-offered must not grow this without limit. Records are
// keyed by core+length and counted, so a repeating conflict stays ONE row with a rising count.
const MAX_RECORDS = 256
const records = new Map()
const watched = new WeakSet()

export function recordConflict ({ discoveryKey, key, length, fork, writable, quorum, store }) {
  const id = discoveryKey + '@' + length
  const existing = records.get(id)
  if (existing) {
    existing.count++
    existing.lastSeenAt = new Date().toISOString()
    return existing
  }
  if (records.size >= MAX_RECORDS) return null
  const rec = {
    discoveryKey,
    key,
    length,
    fork,
    // Whether THIS machine can sign the forked core — the difference between "our key authored two
    // histories" and "someone else's fork arrived here", which is the first question triage asks.
    writable,
    quorum,
    store,
    count: 1,
    firstSeenAt: new Date().toISOString(),
    lastSeenAt: new Date().toISOString()
  }
  records.set(id, rec)
  return rec
}

function attach (core, store) {
  if (watched.has(core)) return
  watched.add(core)

  // A monitor is anything with `emit` and a `_monitorIndex` slot (core.js:79-89). Deliberately NOT a
  // Hypercore session: opening a session per core just to observe would keep every core alive and change
  // the very lifecycle we are trying to observe.
  // The FULL monitor contract, taken from every site that touches `core.monitors`: `emit` (core.js:98,
  // 104, 704-757, 917; replicator.js:3196-3240), `extensions` as a Map — iterated on every peer-add
  // (replicator.js:3199) — and a numeric `padding`, subtracted from byte counts on download/upload
  // (replicator.js:3211, 3220). Omitting the last two is not a silent degradation: it throws inside
  // replication and takes the connection down, which is how the first version of this was caught.
  const monitor = {
    _monitorIndex: -1,
    extensions: new Map(),
    padding: 0,
    emit (name, length, fork) {
      if (name !== 'conflict') return
      recordConflict({
        discoveryKey: hex(core.discoveryKey),
        key: hex(core.key),
        length,
        fork,
        writable: !!(core.header && core.header.keyPair),
        quorum: core.header && core.header.manifest ? core.header.manifest.quorum : null,
        store
      })
      // Keep the upstream line too — it is what every existing incident log was grepped for, and a report
      // that silently replaced it would break the habit before the new surface has earned trust.
      console.log(JSON.stringify({
        msg: 'CONFLICT — core froze and stopped replicating (in-memory only; returns on restart)',
        store,
        discoveryKey: hex(core.discoveryKey),
        key: hex(core.key),
        length,
        fork
      }))
    }
  }

  try { core.addMonitor(monitor) } catch { watched.delete(core) }
}

// Attach to every core this corestore opens, now and later. Namespaces share the root's core tracker, so
// watching the root covers `store.namespace(...)` opens too — no need to watch each namespace.
export function watchStore (store, label) {
  if (!store || typeof store.watch !== 'function') return store
  store.watch((core) => attach(core, label))
  return store
}

// The report `diag` collects. `seenSinceBoot` is stated rather than implied because a zero here means
// "none since this process started", NOT "none ever" — and a restart is exactly when they re-fire.
export function conflictDiag () {
  const all = [...records.values()].sort((a, b) => b.count - a.count)
  return {
    seenSinceBoot: all.length,
    total: all.reduce((n, r) => n + r.count, 0),
    truncated: records.size >= MAX_RECORDS,
    conflicts: all
  }
}

export function _reset () { records.clear() }