// cores — map a hypercore discovery key back to a core, offline, against a LIVE daemon.
//
// WHY THIS EXISTS. When two copies of the same log disagree, hypercore prints
//
// [hypercore] conflict detected in 4a6f1b30cea98c6e… (writable=…,quorum=…)
//
// and that is the ONLY trace it leaves: `core.js:685` is an upstream `// tmp log so we can see these`,
// and nothing in this stack (hpp, atproto-room, autobase 7.28.1) listens for the `conflict` event. Worse,
// the id it prints is the DISCOVERY key — `blake2b("hypercore", key)`, one-way — so you cannot tell which
// core it is by looking. loser-club has been logging conflicts on `4a6f1b30` across restarts with no way
// to name the core.
//
// You do not have to invert the hash. Every corestore already stores the pair: `@corestore/core` is keyed
// BY discovery key and `@core/auth` holds the public key beside it. So this is a lookup, not a search.
//
// SAFE AGAINST A RUNNING DAEMON. Opened `{ readOnly: true }`, which skips the CORESTORE device-file lock
// (hypercore-storage/index.js:498) and opens RocksDB read-only — no lock contention, no `pgh app stop`
// first. Every op here is a read; nothing writes, repairs or fetches.
//
// WHAT IT ANSWERS, in order of what an incident actually needs:
// 1. which core is <dkey> — store dir, public key, corestore alias NAME
// 2. did WE sign it — `writable` means this machine holds the secret key. A conflict on
// a core we can sign means our own key authored two histories; a
// conflict on one we cannot is someone else's fork arriving here.
// 3. where else does it live — the same key writable in TWO stores is the duplicate-writer shape
// that PRODUCES conflicts, and it is otherwise invisible.
//
// Deliberately NOT reported: whether the core is currently frozen. `checkConflict` sets `header.frozen`
// and writes it via `setAuth`, but `@core/auth` has no `frozen` field (encoding/spec/hyperschema:440), so
// it is dropped on encode and never persisted. The freeze is in-memory for the life of the process, which
// is exactly why conflicts re-fire at every re-join. There is nothing on disk to read.
import fs from 'bare-fs'
import os from 'bare-os'
import { join, relative } from 'bare-path'
import b4a from 'b4a'
import flat from 'flat-tree'
import crypto from 'hypercore-crypto'
import CorestoreStorage from 'hypercore-storage'
import { loadConfig } from './config.js'
const hex = (b) => (b ? b4a.toString(b, 'hex') : null)
// The merkle tree hash at a given length — byte-for-byte the value `checkConflict` compares
// (hypercore/lib/core.js:632-634: `crypto.tree(await maybeGetRootsFromStorage(storage, length))`). Two
// copies of one core that hash DIFFERENTLY at the same length are two signed histories, which is the
// literal definition of the conflict hypercore reports. Rebuilt from the public primitives rather than
// deep-importing hypercore's private MerkleTree, so this cannot silently rot against an internal refactor.
//
// null means "cannot say": a sparse replica legitimately lacks the tree nodes for that length, and an
// absent node must never be allowed to read as agreement.
export async function treeHashAt (storage, length) {
if (!length) return null
const indexes = flat.fullRoots(2 * length)
const rx = storage.read()
const pending = indexes.map((i) => rx.getTreeNode(i))
rx.tryFlush()
const roots = await Promise.all(pending)
if (roots.some((r) => !r)) return null
return hex(crypto.tree(roots))
}
// Do the copies of one core AGREE? Answers the question the conflict log cannot: a shorter copy is
// innocent if it is a strict PREFIX of the longer one (ordinary lag) and damning if it is not (a fork).
// Compared at the shortest common length, which is what a peer itself uses when it detects a conflict
// (`Math.min(this.core.state.length, this.remoteLength)`, replicator.js:1462).
//
// On divergence it bisects for the FIRST length that disagrees. That number is the finding: it says where
// the two histories parted, which no aggregate can, and it costs ~log2(length) reads.
async function readLength (core) {
const rx = core.read()
const p = rx.getHead()
rx.tryFlush()
const head = await p
return head ? head.length : null
}
export async function compareCopies (dirs, discoveryKeyHex, { at: pinned = null } = {}) {
const dkey = b4a.from(discoveryKeyHex, 'hex')
const open = []
try {
for (const dir of dirs) {
const s = new CorestoreStorage(dir, { readOnly: true })
await s.ready()
const core = await s.resumeCore(dkey)
if (core) open.push({ dir, s, core, length: await readLength(core) })
else await s.close()
}
if (!open.length) return { comparable: false, reason: 'no store here holds this core' }
const lengths = open.map((o) => o.length).filter((n) => n != null)
// `--at <n>` pins the length. Without it two machines cannot be compared at all: the interesting case
// is precisely when their lengths DIFFER (a 4-block copy on one host, 17341 on another), and a tree
// hash is only meaningful against another hash taken at the SAME length. With one copy per host there
// is no shortest-common length to infer, so the operator has to name it — usually the shorter host's.
const at = pinned || Math.min(...lengths)
const copies = []
for (const o of open) copies.push({ store: o.dir, length: o.length, treeHashAt: at, treeHash: await treeHashAt(o.core, at) })
const known = copies.filter((c) => c.treeHash)
const distinct = [...new Set(known.map((c) => c.treeHash))]
const agree = distinct.length <= 1
// ONE copy on this host is still worth reporting: the hash is the machine-independent fingerprint of a
// history at a length, so running this on two hosts with the same `--at` and diffing the two strings IS
// the cross-machine comparison. Without this a host holding a single copy could say nothing at all,
// which is the common case — a healthy node has exactly one copy of a core.
if (open.length === 1) {
return {
comparable: true,
comparedAtLength: at,
copies,
unreadable: copies.filter((c) => !c.treeHash).map((c) => c.store),
singleCopy: true,
verdict: copies[0].treeHash
? 'SINGLE — one copy here; compare this treeHash against another host run with the same --at'
: 'SINGLE — one copy here, but its tree nodes at this length are not present, so it cannot be hashed',
divergesAt: null
}
}
let divergesAt = null
if (!agree) {
// First length whose hash disagrees between two copies that DO disagree at `at`. Binary search the
// predicate "these two agree at n", which is monotone: agreement can only be lost, never regained.
const a = open.find((o) => copies.find((c) => c.store === o.dir).treeHash === distinct[0])
const b = open.find((o) => copies.find((c) => c.store === o.dir).treeHash === distinct[1])
let lo = 1
let hi = at
while (lo < hi) {
const mid = Math.floor((lo + hi) / 2)
const [ha, hb] = [await treeHashAt(a.core, mid), await treeHashAt(b.core, mid)]
if (ha && hb && ha === hb) lo = mid + 1
else hi = mid
}
divergesAt = { length: lo, between: [a.dir, b.dir] }
}
return {
comparable: true,
comparedAtLength: at,
copies,
unreadable: copies.filter((c) => !c.treeHash).map((c) => c.store),
agree,
// The verdict in the two words an incident actually needs.
verdict: agree
? 'PREFIX — the shorter copies are consistent with the longer; these copies are not a fork'
: 'FORK — copies disagree at the same length: two signed histories for one key',
divergesAt
}
} finally {
for (const o of open) { try { await o.s.close() } catch {} }
}
}
// A corestore on disk is a dir holding a CORESTORE device file next to a `db/` rocks dir — the two paths
// the constructor itself uses. Walking for that (rather than hardcoding) is the point: it finds the stores
// no one is looking for. hpp spreads them over `store/`, `statebase/`, `membrane/`,
// `atproto-repo-overlay/`, `dids/<did16>/…` and a fresh `statebase-resync/<did16>-r<gen>` per resync
// generation — and LEFTOVER stores from an abandoned rebuild are precisely where a forked copy hides.
export function findStores (root, { depth = 5 } = {}) {
const out = []
const walk = (dir, d) => {
let entries
try { entries = fs.readdirSync(dir) } catch { return }
if (entries.includes('CORESTORE') && entries.includes('db')) { out.push(dir); return }
if (d >= depth) return
for (const name of entries) {
const p = join(dir, name)
try { if (fs.statSync(p).isDirectory()) walk(p, d + 1) } catch {}
}
}
walk(root, 0)
return out.sort()
}
// Every core in one store, with the auth + head that name it. Returns [] rather than throwing on a store
// that will not open: a wedged or half-written store must not hide the answer sitting in the other five.
export async function readStore (dir) {
const s = new CorestoreStorage(dir, { readOnly: true })
try {
await s.ready()
// Corestore's human names, so a match reads "this is the autobase local writer" and not just a hash.
// One core can carry several aliases (different namespaces resolving to it), so collect them all.
const names = new Map()
for await (const { alias, discoveryKey } of s.createAliasStream()) {
const k = hex(discoveryKey)
if (!names.has(k)) names.set(k, [])
names.get(k).push(alias.name)
}
const dkeys = []
for await (const { discoveryKey } of s.createCoreStream()) dkeys.push(discoveryKey)
const infos = await s.getInfos(dkeys, { auth: true, head: true, hints: false })
return dkeys.map((dk, i) => {
const info = infos[i] || {}
const auth = info.auth || {}
const head = info.head || null
const dkHex = hex(dk)
return {
store: dir,
discoveryKey: dkHex,
key: hex(auth.key),
names: names.get(dkHex) || [],
// The secret key lives beside the core. Present => this machine can APPEND to it, i.e. any fork of
// this core was authored here. This single field decides whether a conflict is ours or inbound.
writable: !!auth.keyPair,
length: head ? head.length : null,
fork: head ? head.fork : null,
quorum: auth.manifest ? auth.manifest.quorum : null,
signers: auth.manifest && auth.manifest.signers
? auth.manifest.signers.map((x) => hex(x.publicKey))
: []
}
})
} catch (e) {
return [{ store: dir, error: String((e && e.message) || e) }]
} finally {
try { await s.close() } catch {}
}
}
// Same public key present in more than one store. A plain replica legitimately appears twice (a side store
// replicating what the shared store also holds), so that alone is noise — `writableCopies > 1` is not:
// that is two independent places able to sign the SAME key, which is how "two conflicting signatures exist
// for length N" comes to be true in the first place.
export function findDuplicates (cores) {
const byKey = new Map()
for (const c of cores) {
if (!c.key) continue
if (!byKey.has(c.key)) byKey.set(c.key, [])
byKey.get(c.key).push(c)
}
const dups = []
for (const [key, copies] of byKey) {
if (copies.length < 2) continue
const writableCopies = copies.filter((c) => c.writable).length
dups.push({
key,
discoveryKey: copies[0].discoveryKey,
names: [...new Set(copies.flatMap((c) => c.names))],
copies: copies.map((c) => ({ store: c.store, writable: c.writable, length: c.length, fork: c.fork })),
writableCopies,
// Lengths diverging across copies of one key is a fork you can SEE without waiting for two nodes to
// meet. It is suggestive, not proof — a lagging replica is shorter too — so it is flagged, not called.
lengthsDiffer: new Set(copies.map((c) => c.length)).size > 1,
severity: writableCopies > 1 ? 'DUPLICATE-WRITER' : 'replica'
})
}
return dups.sort((a, b) => (b.writableCopies - a.writableCopies))
}
// args = CLI args AFTER the subcommand: [<dkey-prefix>] [--storage <p>] [--all] [--brief]
export async function coresCommand (args = []) {
const cfg = loadConfig()
const flag = (name, dflt) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : dflt }
const storage = flag('--storage', cfg.storage)
const all = args.includes('--all')
const brief = args.includes('--brief')
const compare = args.includes('--compare')
const pinnedAt = Number(flag('--at', 0)) || null
// A prefix, not a full key: the conflict log is usually quoted truncated in an incident write-up, and
// 8 hex chars already identify a core within one machine.
const target = args.find((a) => /^[0-9a-f]{4,64}$/i.test(a) && !a.startsWith('--'))
if (!target && !all) {
console.error('usage: hyper-proto-peer cores [<discovery-key-prefix>] [--compare] [--all] [--storage <path>] [--brief]')
console.error(' --compare: when a core has several copies on disk, say whether they are the SAME')
console.error(' history (a short copy that is a PREFIX is ordinary lag) or a FORK, and at which block')
console.error(' they parted — the merkle comparison hypercore itself makes when it reports a conflict.')
console.error(' --at <n>: hash at length <n> instead of the shortest common one. With ONE copy this is')
console.error(' how you compare ACROSS machines: run it on each host with the same --at and diff the')
console.error(' treeHash strings. Equal = same history to that length; different = a fork.')
console.error(' Maps the discovery key from a `[hypercore] conflict detected in <dkey>` log line back')
console.error(' to a core: which store holds it, its public key, its corestore alias name, and whether')
console.error(' THIS machine can sign it. --all lists every core in every store instead.')
console.error(' Read-only, and safe to run against a running daemon (no lock taken).')
return 1
}
const stores = findStores(storage)
const cores = []
for (const dir of stores) cores.push(...(await readStore(dir)))
const ok = cores.filter((c) => !c.error)
const rel = (c) => ({ ...c, store: relative(storage, c.store) || '.' })
const matches = target
? ok.filter((c) => (c.discoveryKey || '').startsWith(target.toLowerCase()))
: []
const report = {
tool: 'hyper-proto-peer cores',
host: (() => { try { return os.hostname() } catch { return null } })(),
at: new Date().toISOString(),
storage,
stores: stores.map((d) => relative(storage, d) || '.'),
scanned: ok.length,
failed: cores.filter((c) => c.error).map((c) => ({ store: relative(storage, c.store) || '.', error: c.error })),
query: target || null,
// The answer to "which core is this". Empty means the conflicted core is not in any store under this
// storage root — which is itself informative: it is not ours, or it lives under a different root.
matches: target ? matches.map(rel) : undefined,
// --compare: are the copies of this core the SAME history? The conflict log says a fork exists
// somewhere; this says whether THESE copies are it, and at which block they parted.
comparison: compare && matches.length
? await compareCopies(matches.map((c) => c.store), matches[0].discoveryKey, { at: pinnedAt })
: undefined,
duplicates: findDuplicates(ok).map((d) => ({ ...d, copies: d.copies.map((c) => ({ ...c, store: relative(storage, c.store) || '.' })) })),
cores: all && !brief ? ok.map(rel) : undefined
}
console.log(JSON.stringify(report, null, 2))
// Non-zero when a query found nothing, so a scripted sweep across hosts can branch on "not here".
return target && !matches.length ? 1 : 0
}