// E2E autobase consensus flow tests — bare module, no pear-git-v3 wrapping.
//
// Goal: prove (or disprove) the deadlock hypothesis from the prod
// pear-git-v3 incident: signedLength frozen at 59 of coreLength=362, 2
// indexers in linearizer, the boot-writer (f10b54d2…) silent forever.
//
// Three scenarios:
// 1. Two-writer bootstrap → addWriter(indexer) → graduation. Sanity
// check that the canonical pattern works at all.
// 2. Three-writer setup, one indexer goes offline. Remaining two
// should still ratify (majority quorum 2/3).
// 3. Two-writer setup, boot-writer goes offline. Surviving writer
// alone — does it ratify? If no: matches prod state, deadlock
// confirmed. If yes: prod state has a different cause.
import test from 'brittle'
import Autobase from 'autobase'
import Corestore from 'corestore'
import b4a from 'b4a'
import os from '#node:os'
import path from '#node:path'
import fs from '#node:fs'
import { replicateAndSync } from 'autobase-test-helpers'
const TMP_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'autobase-e2e-'))
let storeCounter = 0
function tmpDir () { return path.join(TMP_ROOT, 'store-' + (storeCounter++)) }
async function makeBase ({ key } = {}) {
const store = new Corestore(tmpDir())
await store.ready()
const base = new Autobase(store, key || null, {
apply,
open: (store) => store.get('view', { valueEncoding: 'json' }),
valueEncoding: 'json',
ackInterval: 100 // auto-ack every 100ms (production-canonical trigger for advancing signedLength)
})
await base.ready()
return { store, base }
}
async function settle (bases, ms = 500) {
// Pump background acks for a while.
await replicateAndSync(bases)
await new Promise(r => setTimeout(r, ms))
await replicateAndSync(bases)
}
let APPLY_TRACE = false
async function apply (nodes, view, host) {
for (const node of nodes) {
if (APPLY_TRACE) {
const wkey = node.from && node.from.key ? node.from.key.toString('hex').slice(0, 6) : '?'
const v = node.value
const summary = v && v.add ? `ADD ${v.add.slice(0, 6)} indexer=${!!v.indexer}` : (v && v.ack !== undefined ? `ack:${v.ack}` : JSON.stringify(v).slice(0, 30))
console.log(` apply: from=${wkey} length=${node.length} indexed=${node.indexed} ${summary}`)
}
if (node.value && node.value.add) {
await host.addWriter(b4a.from(node.value.add, 'hex'), { indexer: !!node.value.indexer })
// Force-ack the new writer too so that addHead-driven graduation sees it.
// (Sanity: would explicit ack help?)
// await host.ackWriter(b4a.from(node.value.add, 'hex'))
} else {
await view.append(node.value)
}
}
}
function snapshot (label, base) {
const indexers = base.linearizer.indexers.map(w => ({
key: (w.core && w.core.key ? w.core.key.toString('hex') : '?').slice(0, 12),
length: w.length,
indexed: w.indexed
}))
const localHex = base.local && base.local.key ? base.local.key.toString('hex').slice(0, 12) : '?'
// Reach into _applyState.system for pendingIndexers / indexers.
const sys = base._applyState && base._applyState.system
const pendingHex = sys && sys.pendingIndexers ? sys.pendingIndexers.map(k => k.toString('hex').slice(0, 12)) : []
const sysIndexersHex = sys && sys.indexers ? sys.indexers.map(idx => idx.key.toString('hex').slice(0, 12)) : []
return {
label,
local: localHex,
indexerCount: indexers.length,
activeWriters: base.activeWriters ? base.activeWriters.size : null,
signedLen: base.signedLength,
systemLen: base.system && base.system.core ? base.system.core.length : null,
indexers,
pendingIndexers: pendingHex,
sysIndexers: sysIndexersHex,
writable: base.writable
}
}
function logSnap (s) {
console.log(` ${s.label}: local=${s.local} writable=${s.writable} indexers=${s.indexerCount} signed=${s.signedLen} sysLen=${s.systemLen}`)
for (const i of s.indexers) console.log(` - ${i.key} len=${i.length} indexed=${i.indexed}`)
if (s.pendingIndexers && s.pendingIndexers.length) console.log(` pendingIndexers=[${s.pendingIndexers.join(', ')}]`)
if (s.sysIndexers && s.sysIndexers.length) console.log(` sys.indexers=[${s.sysIndexers.join(', ')}]`)
}
test('SCENARIO 1 — two-writer bootstrap and graduation', async (t) => {
const A = await makeBase()
await A.base.append({ msg: 'op1 from A' })
await A.base.append({ msg: 'op2 from A' })
const B = await makeBase({ key: A.base.key })
await replicateAndSync([A.base, B.base])
console.log('after initial sync:')
logSnap(snapshot('A', A.base))
logSnap(snapshot('B', B.base))
// A authorizes B as indexer.
const bLocalHex = b4a.toString(B.base.local.key, 'hex')
await A.base.append({ add: bLocalHex, indexer: true })
await replicateAndSync([A.base, B.base])
t.ok(B.base.writable, 'B writable after addWriter')
// B appends — its writer-core grows.
await B.base.append({ msg: 'op1 from B' })
await replicateAndSync([A.base, B.base])
// A acks (its append fires _updateIndexer for B's length>0 head).
await A.base.append({ msg: 'op3 from A' })
await replicateAndSync([A.base, B.base])
// One more sync round — graduation propagates over multiple acks.
await A.base.append({ msg: 'op4 from A' })
await B.base.append({ msg: 'op2 from B' })
await replicateAndSync([A.base, B.base])
const sa = snapshot('A', A.base)
const sb = snapshot('B', B.base)
console.log('after graduation:')
logSnap(sa); logSnap(sb)
t.is(sa.indexerCount, 2, 'A: 2 indexers in linearizer')
t.is(sb.indexerCount, 2, 'B: 2 indexers in linearizer')
t.ok(sa.signedLen >= 4, 'A: signedLen advanced past bootstrap (was 0/1, now ≥4)')
t.ok(sb.signedLen >= 4, 'B: signedLen advanced past bootstrap')
t.teardown(async () => {
await A.base.close(); await B.base.close()
await A.store.close(); await B.store.close()
})
})
// Helper: sync until predicate true. Calls base.ack() each round which is
// the canonical trigger to advance signedLength when pending indexers exist
// (autobase auto-acks on a timer in production via _ackTimer).
async function syncUntil (bases, predicate, { rounds = 30, label = '' } = {}) {
for (let i = 0; i < rounds; i++) {
for (const base of bases) {
try { await base.ack() } catch {}
}
await replicateAndSync(bases)
if (predicate()) return i
if (i % 2 === 1) {
console.log(` syncUntil(${label}) round ${i + 1}:`)
for (const base of bases) {
const s = snapshot(base.local && base.local.key ? base.local.key.toString('hex').slice(0, 6) : '?', base)
console.log(` ${s.label} indexers=${s.indexerCount} signed=${s.signedLen} sysLen=${s.systemLen} pending=[${s.pendingIndexers.join(',')}] sys.idx=[${s.sysIndexers.join(',')}]`)
}
}
}
throw new Error(`syncUntil timed out after ${rounds} rounds — ${label}`)
}
test('SCENARIO 2 — adding a 3rd indexer to a stable 2-indexer base BLOCKS signing', async (t) => {
// This documents a v7.27.3 autobase property critical to the prod incident:
// once 2 writers have been graduated to indexers and signing is happening,
// adding a 3rd indexer raises sys.indexers.length=3 but linearizer.indexers
// stays at 2. shouldMigrate() detects the manifest mismatch but the
// migration requires consensus to advance to incorporate the new manifest —
// and consensus is gated on the very signing that the manifest mismatch
// blocks. signedLength freezes at the addWriter op.
//
// Implication for prod: the device-invite ceremony in lib/handlers/devices.js
// that adds linked devices as role:'indexer' is the trigger. Once the third
// device was added, the autobase entered this stuck state.
const A = await makeBase()
await A.base.append({ msg: 'op1 from A' })
const B = await makeBase({ key: A.base.key })
const C = await makeBase({ key: A.base.key })
await replicateAndSync([A.base, B.base, C.base])
// Graduate B fully — get to a stable 2-indexer signing state.
await A.base.append({ add: b4a.toString(B.base.local.key, 'hex'), indexer: true })
await replicateAndSync([A.base, B.base, C.base])
await syncUntil([A.base, B.base, C.base],
() => A.base.linearizer.indexers.length === 2 && B.base.linearizer.indexers.length === 2,
{ label: 'B graduates', rounds: 10 })
const signedAt2Indexers = A.base.signedLength
console.log(` stable 2-indexer state: signedLen=${signedAt2Indexers}, indexerCount=${A.base.linearizer.indexers.length}`)
// Add C as 3rd indexer.
await A.base.append({ add: b4a.toString(C.base.local.key, 'hex'), indexer: true })
await replicateAndSync([A.base, B.base, C.base])
// Drive 20 rounds of acks. signedLen should not advance because the
// pending manifest migration to 3 signers can't complete.
for (let i = 0; i < 20; i++) {
for (const base of [A.base, B.base, C.base]) {
try { await base.ack() } catch {}
}
await replicateAndSync([A.base, B.base, C.base])
}
const sa = snapshot('A', A.base)
const sb = snapshot('B', B.base)
const sc = snapshot('C', C.base)
console.log('after 20 rounds with 3rd indexer pending:')
logSnap(sa); logSnap(sb); logSnap(sc)
// Hypothesis: linearizer.indexers stays 2 even though sys.indexers is 3,
// and signedLen never advances past the addWriter(C) point.
t.is(sa.indexerCount, 2, 'A: linearizer still has only 2 indexers (3rd never graduates)')
t.is(sa.sysIndexers.length, 3, 'A: sys.indexers HAS all 3 (manifest mismatch)')
t.ok(sa.signedLen <= signedAt2Indexers + 5, 'A: signedLen barely advanced past addWriter(C) op')
t.teardown(async () => {
await A.base.close(); await B.base.close(); await C.base.close()
await A.store.close(); await B.store.close(); await C.store.close()
})
})
test('SCENARIO 3 — two writers, boot-writer drops (does survivor ratify alone?)', async (t) => {
const A = await makeBase()
await A.base.append({ msg: 'op1 from A' })
const B = await makeBase({ key: A.base.key })
await replicateAndSync([A.base, B.base])
await A.base.append({ add: b4a.toString(B.base.local.key, 'hex'), indexer: true })
await replicateAndSync([A.base, B.base])
await B.base.append({ msg: 'op1 from B' })
await A.base.append({ msg: 'op2 from A' })
await replicateAndSync([A.base, B.base])
await B.base.append({ msg: 'op2 from B' })
await A.base.append({ msg: 'op3 from A' })
await replicateAndSync([A.base, B.base])
console.log('after 2-way graduation:')
logSnap(snapshot('A', A.base))
logSnap(snapshot('B', B.base))
const signedBeforeDrop = B.base.signedLength
t.is(snapshot('B', B.base).indexerCount, 2, 'B: 2 indexers before drop')
// A drops. B is alone. With strict majority 2/2, B cannot ratify on its own.
// (This is the prod hypothesis.)
await A.base.close(); await A.store.close()
// Give B time to discover A is gone.
await new Promise(r => setTimeout(r, 200))
await B.base.append({ msg: 'op3 from B (after A drop)' })
await B.base.append({ msg: 'op4 from B' })
await B.base.append({ msg: 'op5 from B' })
await B.base.update()
await new Promise(r => setTimeout(r, 500))
await B.base.update()
const sb = snapshot('B', B.base)
console.log('after A dropped:')
logSnap(sb)
// EXPECTED (hypothesis): signedLen does NOT advance past signedBeforeDrop.
// If this assertion fails, the prod scenario isn't a quorum deadlock and
// we need a different recovery theory.
if (sb.signedLen === signedBeforeDrop) {
console.log(' → CONFIRMED: 2/2 quorum strict, survivor cannot ratify alone (matches prod deadlock)')
t.is(sb.signedLen, signedBeforeDrop, 'B alone: signedLen frozen (deadlock confirmed)')
} else {
console.log(` → SURPRISE: signedLen advanced from ${signedBeforeDrop} to ${sb.signedLen} alone — autobase has a fast-forward I missed`)
t.fail('survivor advanced alone — hypothesis wrong, investigate further')
}
t.teardown(async () => {
await B.base.close(); await B.store.close()
})
})