// mirror.js — hold the bytes of feeds you follow, before anyone asks to read them.
//
// ⛔ THE OUTAGE, measured 2026-09-08. tabertron held five pointers from ashatron's feed and could
// read none of them. ashatron is a laptop, it went offline, and it was the only machine holding the
// bytes. Nothing was broken: a record's bytes are fetched only when a reader asks for them, nobody
// had asked, so no second copy existed. The surface reported it honestly — "the log arrived, the
// bytes did not" — and all five resolved the moment ashatron came back.
//
// ★ SO THIS IS ONLY A TRIGGER. Two things already work and are deliberately NOT rebuilt here:
//
// 1. A fetch WRITES the record into that feed's working tree, so the copy is durable.
// 2. hyper-overlay RE-SEEDS on fetch — "register the verified blob locally so this node now
// SERVES these chunks to other peers too" — so holding implies serving, automatically.
//
// Reading a record once already makes this node a mirror of it. Nothing reads unprompted, and that
// is the entire gap. hyper-overlay's `mirrorSpool` is NOT needed for us: it exists for "a node with
// no working tree of its own", and we have one per feed.
//
// ⛔ FOLLOWED FEEDS ONLY. `peer.timeline` already excludes peeked feeds because including them
// "would make reading a stranger's page indistinguishable from following them". Mirroring a peek
// would be that same collapse with disk attached — every profile glanced at becomes a permanent
// copy. Following is the decision to keep someone's bytes; peeking is not.
//
// Design: hyper-social/docs/HOLDING-AND-TELLING.md
import { embeds } from 'favela-markdown/lib/render.js'
const DEFAULT_CONCURRENCY = 4
/**
* @param feeds () => [{ key, feed }] — the FOLLOWED feeds, supplied by the caller so this
* module never has to know how following is decided.
* @param concurrency files in flight. ⚠ Bounds FILES, not chunks: the layer below already holds an
* RTT-scaled inflight window per peer, and re-implementing that would be worse
* than what exists. What is unbounded is the file count, and a freshly followed
* feed is all of them at once.
*/
export function createMirror ({ feeds, files = null, concurrency = DEFAULT_CONCURRENCY, debounceMs = 2000, log = () => {} }) {
if (typeof feeds !== 'function') throw new Error('createMirror: feeds() is required')
let stopped = false
let running = false
let timer = null
let lastPassAt = 0
// ⛔ Counted per TRIGGER so a dead hook is OBSERVABLE. 0.0.40's watcher never fired and nothing
// on a running node could say so — the only evidence was how LONG a record took to appear, which
// is inference. These counters are what turned that into a measurement.
const fired = { append: 0, connection: 0, boot: 0, files: 0 }
const bases = []
const hooked = new Map()
// Attach an 'append' listener to every writer core we have not already hooked. A hypercore emits
// append when blocks land, INCLUDING replicated ones — the earliest honest signal someone else
// wrote something. Idempotent, so calling it every pass is free after the first.
function rehook (onAppend) {
for (const base of bases) {
let writers
try { writers = base.writers } catch { continue }
if (!writers) continue
try {
for (const w of writers) {
const core = w && w.core
if (!core || typeof core.on !== 'function') continue
// ⛔ Keyed on the CORE OBJECT, not on core.id or core.key. Keying on an identifier meant
// a core that had not yet exposed one was silently skipped — and "silently skipped" is
// precisely the failure this whole trigger has already had three times.
if (hooked.has(core)) continue
core.on('append', onAppend)
hooked.set(core, onAppend)
}
} catch { /* a writer set mid-change is not a failure of the pass */ }
}
}
/**
* ⛔ LEVEL 2 — the files a record NAMES.
*
* A record's own bytes are one oid, carried in the log. The pictures and files inside it are more
* oids, carried in its markdown. Mirroring only the first leaves a post you can read whose image
* will not load — the same failure as the outage this module exists for, one level down.
* Following means holding what somebody published, and a picture is part of what they published.
*
* ⚠ Both levels are the SAME store. files.js: "There is NO blob store" — a record and an image
* are both registerFile/fetchFile by content hash in hyper-overlay. There is one store here.
*
* `embeds()` is favela-markdown's real parser over the markdown tree, so this sees exactly what a
* renderer would and never a regex's idea of it. A file we cannot fetch does NOT undo the record:
* the record is held and readable, and the picture is missing — which is the honest state one
* level down, and the next pass retries it.
*/
async function mirrorFiles (record) {
if (!files || typeof files.get !== 'function') return
const body = record && record.body
if (!body) return
let refs
try { refs = embeds(body) } catch { return }
for (const e of refs) {
if (stopped) return
try {
if (await files.get(e.oid)) fired.files++
} catch { /* missing, not fatal — the record stands */ }
}
}
async function mirrorFeed (key, feed) {
// ⛔ `pointers()` and not `list()` or `unresolved()`. Both of those RESOLVE every record, which
// means both of them fetch — unbounded, all at once. Enumerating without fetching is what lets
// this pass be bounded at all.
let rows
try {
// ⛔ REFRESH THE VIEW FIRST. `pointers()` reads the linearised view, and the view only
// advances when apply runs. Replicated blocks can be sitting in the writer cores with the
// view still behind them — so a sweep that skipped this would keep reporting "nothing new"
// while the log had in fact grown. This is what makes the pass correct no matter what
// triggered it.
if (typeof feed.update === 'function') await feed.update()
rows = await feed.pointers()
} catch (err) {
// A feed whose view is not ready yet is not a failure of the pass. It will be there next time.
log({ msg: 'mirror: could not list feed', feed: key, error: err.message })
return { fetched: 0, failed: 0 }
}
// Ask the STORE, not the log: `get` is a local read and answers without touching the network,
// so a record already on disk costs nothing and is skipped before any slot is taken.
const missing = []
for (const r of rows) {
if (stopped) break
try {
if ((await feed.store.get(r.collection, r.name)) == null) missing.push(r)
} catch { missing.push(r) }
}
let fetched = 0
let failed = 0
let next = 0
const worker = async () => {
while (!stopped) {
const i = next++
if (i >= missing.length) return
const r = missing[i]
try {
// ⛔ One unfetchable record must not end the pass. A peer that is away is not a peer that
// is gone: the record stays unresolved, the surface says so, and the next pass retries.
const got = await feed.store.ensureLocal(r.collection, r.name, r.pointer && r.pointer.$oid)
if (got == null) failed++
else { fetched++; await mirrorFiles(got) }
} catch {
failed++
}
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, missing.length) }, worker))
if (fetched || failed) log({ msg: 'mirror', feed: key, fetched, failed, missing: missing.length })
return { fetched, failed }
}
return {
/** One sweep over every followed feed. Safe to call concurrently — a second call is a no-op. */
async pass () {
if (stopped || running) return { fetched: 0, failed: 0 }
running = true
// Hook whatever writers exist NOW. On the first pass a remote feed usually has none; by the
// second it does, and from then on every remote append triggers a sweep of its own.
rehook(() => this.schedule('append'))
let fetched = 0
let failed = 0
try {
for (const { key, feed } of feeds()) {
if (stopped) break
const r = await mirrorFeed(key, feed)
fetched += r.fetched
failed += r.failed
}
} finally {
running = false
lastPassAt = Date.now()
}
return { fetched, failed }
},
/**
* Sweep again shortly, coalescing a burst into one pass.
*
* ⛔ THE BOOT PASS ALONE IS NOT ENOUGH, and an end-to-end test on two machines is what proved
* it. At boot a followed feed's log has usually NOT replicated yet — the swarm connection is
* still forming — so the sweep runs, correctly finds nothing to do, and never runs again. The
* unit tests missed this because a fake hands back its pointers instantly; nothing simulated a
* log that arrives thirty seconds later. Records must be fetched when the log ADVANCES, which
* is the only moment we know there is something new to fetch.
*/
schedule (why = 'append') {
if (stopped || timer) return
fired[why] = (fired[why] || 0) + 1
timer = setTimeout(() => {
timer = null
this.pass().catch(() => {})
}, debounceMs)
if (timer && timer.unref) timer.unref()
},
/**
* Sweep whenever `base` advances. Autobee emits 'update' on every view change, which is far
* more often than the log actually gains records — hence the debounce, and hence `schedule`
* rather than `pass` directly.
*/
watch (base) {
if (!base) return () => {}
bases.push(base)
// ⛔ REGISTERED, NOT HOOKED — hooking happens in every pass. `base.writers` is an
// ActiveWriters whose `active` map is EMPTY when a freshly-opened remote feed returns from
// hold(): writers are only added once the system core has replicated. Hooking once here
// therefore attached to NOTHING, silently, and that is the third time in this feature that
// doing a thing once at open time ran before the data it needed existed. Re-hooking per pass
// costs a Map lookup and cannot be early.
return () => { bases.length = 0 }
},
/**
* A peer connected — retry what we could not fetch before.
*
* ⛔ THIS IS THE EVENT THE TIMER WAS STANDING IN FOR. A record stays unresolved because nobody
* holding its bytes is online. Their coming back is a swarm 'connection', not an absence, and
* re-checking it on a clock was polling for data — the one thing the rule is absolute about.
*/
onPeer () { this.schedule('connection') },
/**
* Which trigger is doing the work. `hooked` is the one that matters: 0.0.42 hooked ZERO cores
* and looked identical to a working feature from the outside.
*/
stats () { return { ...fired, hooked: hooked.size, bases: bases.length, lastPassAt } },
stop () {
stopped = true
if (timer) { clearTimeout(timer); timer = null }
for (const [c, fn] of hooked) { try { c.off('append', fn) } catch {} }
hooked.clear()
bases.length = 0
}
}
}
export default { createMirror }