// Writer-set scaling under autobee — does a many-writer log cost every reader?
//
// The question driving it: a "thread" where N people like a post is an N-writer log. Under autobase
// a reader needs every writer's core to compute the view, which is the wall the whole feed design
// was built to dodge. autobee has fast-forward over oplog heads, so the cost MAY be different.
// This measures it instead of arguing about it.
//
// Topology: one creator (genesis indexer) + N writers, each its own Corestore (own primaryKey → own
// local writer key). Every writer is piped to the creator only — a star, which is the realistic
// shape (you connect to whoever is around, not to all N).
//
// Then a FRESH reader opens the base by key, replicates from the creator ALONE, and we measure what
// it costs it to reach the full view.
import fs from '#node:fs'
import os from '#node:os'
import process from '#node:process'
import { join } from '#node:path'
import b4a from 'b4a'
import Corestore from 'corestore'
import Autobase from 'autobase'
import Hyperbee from 'hyperbee'
const enc = (v) => b4a.from(JSON.stringify(v), 'utf8')
const dec = (b) => JSON.parse(b4a.toString(b, 'utf8'))
const tmp = () => fs.mkdtempSync(join(os.tmpdir(), 'scale-'))
function duBytes (dir) {
let total = 0
const walk = (d) => {
let ents
try { ents = fs.readdirSync(d, { withFileTypes: true }) } catch { return }
for (const e of ents) {
const p = join(d, e.name)
if (e.isDirectory()) walk(p)
else { try { total += fs.statSync(p).size } catch {} }
}
}
walk(dir)
return total
}
// The SAME experiment against autobase 7.28.1. Three API differences, nothing else changes:
// open receives a STORE (autobee hands you the bee); the view is a Hyperbee with put/get rather
// than a write batch; and the writer key is node.from.key, not node.key.
async function apply (nodes, view, host) {
for (const node of nodes) {
let op
try { op = dec(node.value) } catch { continue }
if (op.t === 'add-writer') { try { await host.addWriter(b4a.from(op.key, 'hex'), { indexer: false }) } catch {} ; continue }
if (op.t === 'like') await view.put('like/' + op.by, enc({ by: op.by }))
}
}
const open = (store) => new Hyperbee(store.get('view'), { keyEncoding: 'utf-8', valueEncoding: 'binary', extension: false })
function pipePair (a, b) {
const s1 = a.replicate(true)
const s2 = b.replicate(false)
s1.pipe(s2).pipe(s1)
return () => { try { s1.destroy() } catch {}; try { s2.destroy() } catch {} }
}
async function until (fn, ms = 60000, poll = 20) {
const end = Date.now() + ms
while (Date.now() < end) {
if (await fn()) return true
await new Promise((r) => setTimeout(r, poll))
}
return false
}
async function countLikes (db) {
let n = 0
for await (const node of db.view.createReadStream({ gte: 'like/', lt: 'like0' })) n++
return n
}
async function run (N) {
const t0 = Date.now()
const creatorDir = tmp()
const creator = new Autobase(new Corestore(creatorDir), null, { apply, open, valueEncoding: 'binary' })
await creator.ready()
const writers = []
const unpipes = []
for (let i = 0; i < N; i++) {
const dir = tmp()
const w = new Autobase(new Corestore(dir), creator.key, { apply, open, valueEncoding: 'binary' })
await w.ready()
writers.push({ w, dir })
unpipes.push(pipePair(creator, w))
}
// Admit them all, then have each append one "like".
const tAdmit = Date.now()
for (const { w } of writers) await creator.append(enc({ t: 'add-writer', key: b4a.toString(w.local.key, 'hex') }))
await creator.update()
const admitted = await until(async () => {
for (const { w } of writers) { await w.update(); if (!w.writable) return false }
return true
}, 120000)
const admitMs = Date.now() - tAdmit
if (!admitted) return { N, error: 'not all writers became writable' }
const tWrite = Date.now()
for (let i = 0; i < N; i++) await writers[i].w.append(enc({ t: 'like', by: String(i) }))
await until(async () => { await creator.update(); return (await countLikes(creator)) === N }, 120000)
const writeMs = Date.now() - tWrite
// ★ flush() — "wait until all known writers have been fully indexed". The README says each flush
// STAMPS the head you vouch for into your own oplog, and peers read those stamps. Without it there
// may be no head for a joiner to fast-forward to, which would make fastForwards:0 a usage error
// rather than a defect.
const tFlush = Date.now()
if (creator.flush) await creator.flush()
const flushMs = Date.now() - tFlush
const creatorBytes = duBytes(creatorDir)
const creatorCores = creator.store ? creator.store.cores.size : -1
// ⛔ Tear the writers DOWN before measuring the reader. The first version of this measured a
// reader competing with N live autobee instances on one event loop and called the result
// "reader catch-up" — that number was the harness, not the system.
for (const u of unpipes) u()
for (const { w } of writers) await w.close()
// A FRESH reader, connected to the creator ALONE, with nothing else running.
// Run it twice: fast-forward as configured by default, and explicitly disabled, so the mechanism
// is OBSERVED rather than assumed. stats.fastForwards says which path actually ran.
const measureReader = async (ffOpt, poll = 20) => {
const dir = tmp()
const store = new Corestore(dir)
const t = Date.now()
const opts = ffOpt === false ? { apply, open, valueEncoding: 'binary', fastForward: false }
: { apply, open, valueEncoding: 'binary' }
const r = new Autobase(store, creator.key, opts)
await r.ready()
const un = pipePair(creator, r)
const ok = await until(async () => { await r.update(); return (await countLikes(r)) === N }, 120000, poll)
const ms = Date.now() - t
const out = {
ms,
kb: Math.round(duBytes(dir) / 1024),
cores: store.cores ? store.cores.size : -1,
ff: r.fastForwardTo === null ? 0 : 1,
applies: -1,
drains: -1,
ok
}
un()
await r.close()
return out
}
// A/B the FLAG and the HARNESS independently. If the slow column tracks the poll interval rather
// than the flag, the cost is mine.
const ffOn = await measureReader(undefined, 20)
const ffOnSlowPoll = await measureReader(undefined, 500)
const ffOff = await measureReader(false, 20)
await creator.close()
return { N, flushMs, admitMs, writeMs, creatorKB: Math.round(creatorBytes / 1024), creatorCores, ffOn, ffOnSlowPoll, ffOff, totalMs: Date.now() - t0 }
}
const sizes = (process.argv[2] || '5,25,100').split(',').map(Number)
console.log('N\t| ff=on poll20: ms ffs applies\t| ff=on poll500: ms ffs applies\t| ff=off poll20: ms ffs applies')
for (const N of sizes) {
const r = await run(N)
if (r.error) { console.log(N + '\tERROR ' + r.error); continue }
const f = (x) => [x.ms, x.ff, x.applies].join(' ')
console.log([r.N, 'flush=' + r.flushMs, '| ' + f(r.ffOn), '| ' + f(r.ffOnSlowPoll), '| ' + f(r.ffOff)].join('\t'))
}
process.exit(0)