OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-store commits / 5ab5bfe

0.0.26: ensureLocal never destroys bytes a pointer may still name
THE INCIDENT (loser-club, stortron, 2026-10-14): an operator update of app.bsky.labeler.service/self wrote the new bytes (B) and appended the pointer, but before the linearized view showed it the sign gate called ensureLocal with the OLD pointer (A). The local file (B) did not match A, so _refetchOver replaced it with A from the co-owner. Then B's pointer converged and no node held B: both co-owners' sign gates stalled ("pointer present, bytes unreplicated") until a further write superseded it. A local mismatch can be BEHIND (a stale copy: replace it, unchanged — see ensure-local-verify) or AHEAD (our own write, the view lagging). The store cannot tell, so it no longer decides by destroying: - _keep: before replacing a record file, copy its bytes to superseded/<oid>.dagcbor (beside records/, never inside it); newest 256 kept. - _fromKept: ensureLocal checks the kept copy of the requested oid BEFORE the network (mismatch and absent paths); installed only if it hashes to the oid, via temp + rename, keeping whatever it replaces. Test first: test/ensure-local-no-loss.test.js reproduces the incident with a realistic overlay (an oid is servable only while some file still holds it): B was lost (null) before, is restored now; an unverifiable kept copy is never installed. Suite 88/88. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5ab5bfe385bafaf55e0737f7ec44ea9f14260f3b
by plankatron <fleeky@gmail.com> · 2026-10-14T01:43:20.000Z

Changed files (3) +128 -1

lib/overlay-repo-store.js +53 -0
……@@ -22,6 +22,7 @@ import { join } from '#node:path'
2222import { tidNow } from './tid.js'
2323
2424const EXT = '.dagcbor'
25const KEEP_MAX = 256 // superseded/ entries kept (see _keep)
2526const byteOrder = (a, b) => b4a.compare(b4a.from(a), b4a.from(b)) // match Hyperbee's key comparison exactly
2627// atproto collection NSIDs + rkeys are single path segments (no '/'); reject anything else so a
2728// record key can never escape the records/ dir or collide with a parent path.
……@@ -34,6 +35,9 @@ export class OverlayRepoStore {
3435 if (typeof root !== 'string') throw new Error('OverlayRepoStore: root dir path required')
3536 this._root = root
3637 this._recordsDir = join(root, 'records')
38 // Bytes a re-fetch REPLACED, kept content-addressed (<oid>.dagcbor) beside records/ — never inside it,
39 // where records() would read the dir as a collection. See _keep / _fromKept.
40 this._keptDir = join(root, 'superseded')
3741 this._headFile = join(root, 'head.json')
3842 this._overlay = opts.overlay || null
3943 this._did = opts.did || null
……@@ -163,6 +167,10 @@ export class OverlayRepoStore {
163167 // unchanged wherever verification is impossible.
164168 if (!h || h.oid === oid) return local // verified against the converged pointer — the common path
165169 this._logStaleLocal(collection, rkey, oid, h ? h.oid : null)
170 // The pointer may name bytes THIS node once held and replaced (a newer local write the lagging view
171 // had overwritten — loser-club 2026-10-14). Kept copies are verified and installed before the network.
172 const kept = await this._fromKept(collection, rkey, oid)
173 if (kept != null) return kept
166174 const fetched = await this._refetchOver(collection, rkey, oid)
167175 // Deliberately null, NOT the stale record, when the authoritative bytes cannot be obtained.
168176 // Returning the superseded copy is what published wrong content: this record then reads as
……@@ -174,6 +182,8 @@ export class OverlayRepoStore {
174182 return fetched
175183 }
176184 if (!this._overlay || !oid) return null
185 const kept = await this._fromKept(collection, rkey, oid)
186 if (kept != null) return kept
177187 const disk = this._pathFor(collection, rkey)
178188 try {
179189 fs.mkdirSync(join(this._recordsDir, collection), { recursive: true })
……@@ -183,6 +193,45 @@ export class OverlayRepoStore {
183193 return this.get(collection, rkey)
184194 }
185195
196 // Keep the bytes currently at `disk` under superseded/<oid>.dagcbor before they are replaced. Bounded:
197 // only the newest KEEP_MAX entries stay (records are small; this exists to bridge a view that lags a
198 // local write by seconds, not to archive history). Best-effort — a failure here never blocks the repair.
199 async _keep (disk) {
200 try {
201 const h = await this._overlay.hashFile(disk)
202 if (!h || !h.contentHash) return
203 fs.mkdirSync(this._keptDir, { recursive: true })
204 const dest = join(this._keptDir, h.contentHash + EXT)
205 if (!fs.existsSync(dest)) fs.copyFileSync(disk, dest)
206 const entries = fs.readdirSync(this._keptDir).filter((n) => n.endsWith(EXT))
207 if (entries.length > KEEP_MAX) {
208 const byAge = entries.map((n) => ({ n, t: fs.statSync(join(this._keptDir, n)).mtimeMs })).sort((a, b) => a.t - b.t)
209 for (const { n } of byAge.slice(0, entries.length - KEEP_MAX)) { try { fs.unlinkSync(join(this._keptDir, n)) } catch {} }
210 }
211 } catch {}
212 }
213
214 // Install the kept copy of `oid` as this record's file, if we have one AND it hashes to `oid`. Same
215 // temp-then-rename discipline as _refetchOver: the current file is untouched unless the copy verifies.
216 async _fromKept (collection, rkey, oid) {
217 if (!this._overlay || typeof this._overlay.hashFile !== 'function' || !/^[0-9a-f]{16,}$/i.test(String(oid || ''))) return null
218 const src = join(this._keptDir, oid + EXT)
219 if (!fs.existsSync(src)) return null
220 const disk = this._pathFor(collection, rkey)
221 const tmp = disk + '.kept-' + String(oid).slice(0, 12)
222 try {
223 fs.mkdirSync(join(this._recordsDir, collection), { recursive: true })
224 fs.copyFileSync(src, tmp)
225 const h = await this._overlay.hashFile(tmp)
226 if (!h || h.contentHash !== oid) { try { fs.unlinkSync(tmp) } catch {}; return null }
227 if (fs.existsSync(disk)) await this._keep(disk)
228 fs.renameSync(tmp, disk)
229 try { await this._overlay.registerFile(this._overlayPath(collection, rkey), disk, { size: h.size }) } catch {}
230 try { console.log(JSON.stringify({ msg: 'ensureLocal: restored record bytes this node had kept (a write the view had lagged)', did: this._did, key: collection + '/' + rkey, oid: String(oid).slice(0, 16) })) } catch {}
231 return this.get(collection, rkey)
232 } catch { try { fs.unlinkSync(tmp) } catch {}; return null }
233 }
234
186235 // Replace a SUPERSEDED local record file with the authoritative bytes for `oid`, without destroying
187236 // what we hold until the replacement is verified. Fetches to a sibling temp path (reSeed:false so the
188237 // overlay never indexes the temp), re-asserts the hash, then renames into place atomically and
……@@ -201,6 +250,10 @@ export class OverlayRepoStore {
201250 this._logFetchMiss(collection, rkey, oid, 'fetched bytes did not hash to the requested oid')
202251 return null
203252 }
253 // ⛔ KEEP WHAT WE REPLACE. "Local does not match the pointer" can mean we are BEHIND (a stale copy) or
254 // AHEAD (our own write; the view has not linearized its pointer yet). We cannot tell which, so the
255 // outgoing bytes are kept content-addressed and a later pointer to them resolves from _fromKept.
256 await this._keep(disk)
204257 fs.renameSync(tmp, disk)
205258 try { await this._overlay.registerFile(this._overlayPath(collection, rkey), disk, { size: h.size }) } catch {}
206259 try {
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "atproto-store",
3 "version": "0.0.25",
3 "version": "0.0.26",
44 "description": "Holepunch-native atproto repo store — records in a classic Hyperbee (collection/rkey -> DAG-CBOR). The P2P home; the atproto-repo module is the bridge serializer over it.",
55 "type": "module",
66 "main": "lib/repo-store.js",
test/ensure-local-no-loss.test.js +74 -0
……@@ -0,0 +1,74 @@
1// ensureLocal must never DESTROY bytes the converged pointer may still name.
2//
3// THE INCIDENT (loser-club, stortron, 2026-10-14). An operator `update` of app.bsky.labeler.service/self:
4// put() wrote the new bytes (B) and appended the pointer — but BEFORE the linearized view showed it, a
5// reader (the sign gate) called ensureLocal with the OLD converged pointer (A). The local file (B) did not
6// match A, so it was treated as a superseded copy and atomically REPLACED with A, fetched from the
7// co-owner. Then B's pointer converged — and no node held B anymore. Both co-owners' sign gates stalled
8// ("pointer present, bytes unreplicated") until a further write superseded it.
9//
10// A local mismatch can be "behind" (a stale copy — replace it, ensure-local-verify.test.js) OR "ahead"
11// (our own write, the view not yet caught up). The store cannot tell which. So it must not choose by
12// destroying: the bytes it replaces are kept, content-addressed, and a later pointer to them resolves
13// locally instead of stalling.
14//
15// Run: brittle-node test/ensure-local-no-loss.test.js
16import test from 'brittle'
17import fs from 'fs'
18import os from 'os'
19import path from 'path'
20import crypto from 'hypercore-crypto'
21import { OverlayRepoStore } from '../lib/overlay-repo-store.js'
22
23const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'elnl-'))
24const LABELER = 'app.bsky.labeler.service'
25const A = { $type: LABELER, createdAt: '2026-10-14T01:19:49.000Z', policies: { labelValues: [] } }
26const B = { $type: LABELER, createdAt: '2026-10-14T01:23:30.000Z', policies: { labelValues: ['spam'] } }
27
28// A REALISTIC overlay: it serves an oid only while some registered path STILL holds bytes hashing to it
29// (a real overlay indexes paths; overwrite the file and those bytes are gone from the network too).
30function realisticOverlay (peers = []) {
31 const ov = {
32 paths: new Map(), // diskPath -> overlayPath
33 async registerFile (overlayPath, diskPath) { ov.paths.set(diskPath, overlayPath); return { contentHash: hash(fs.readFileSync(diskPath)) } },
34 async hashFile (p) { try { const b = fs.readFileSync(p); return { contentHash: hash(b), size: b.length } } catch { return null } },
35 // What this node can serve right now.
36 servable (oid) { for (const p of ov.paths.keys()) { try { const b = fs.readFileSync(p); if (hash(b) === oid) return b } catch {} } return null },
37 async fetchFile (oid, o = {}) {
38 for (const peer of peers) { const b = peer.servable(oid); if (b) { fs.writeFileSync(o.destPath, b); return { destPath: o.destPath, size: b.length } } }
39 return null
40 }
41 }
42 return ov
43}
44const hash = (b) => crypto.data(b).toString('hex')
45
46test('⛔ a newer local write clobbered by a lagging pointer is RESTORED when its own pointer converges', async (t) => {
47 // The co-owner holds A (replicated earlier); we hold A, then write B.
48 const coOwner = realisticOverlay()
49 const theirs = await new OverlayRepoStore(tmp(), { overlay: coOwner }).ready()
50 const { oid: oidA } = await theirs.put(LABELER, 'self', A)
51
52 const ours = await new OverlayRepoStore(tmp(), { overlay: realisticOverlay([coOwner]) }).ready()
53 await ours.put(LABELER, 'self', A)
54 const { oid: oidB } = await ours.put(LABELER, 'self', B) // the operator update
55 t.not(oidA, oidB)
56
57 // The view still shows A's pointer: the stale-copy rule replaces our file with A (unchanged behaviour).
58 t.alike(await ours.ensureLocal(LABELER, 'self', oidA), A, 'the lagging pointer resolves to A')
59
60 // Now B's pointer converges. Nobody on the network holds B any more — only what WE kept.
61 const got = await ours.ensureLocal(LABELER, 'self', oidB)
62 t.alike(got, B, 'B is restored from the bytes this node kept, instead of stalling the chain')
63 t.is((await ours.localOid(LABELER, 'self')).oid, oidB, 'and the record file holds B again')
64})
65
66test('the kept bytes are verified: a cache entry that does not hash to the oid is never installed', async (t) => {
67 const store = await new OverlayRepoStore(tmp(), { overlay: realisticOverlay() }).ready()
68 const { oid: oidA } = await store.put(LABELER, 'self', A)
69 const keptDir = path.join(store._root, 'superseded')
70 fs.mkdirSync(keptDir, { recursive: true })
71 fs.writeFileSync(path.join(keptDir, 'f'.repeat(64) + '.dagcbor'), Buffer.from('not the bytes'))
72 t.is(await store.ensureLocal(LABELER, 'self', 'f'.repeat(64)), null, 'unverifiable → unresolved (the sign gate fails safe)')
73 t.is((await store.localOid(LABELER, 'self')).oid, oidA, 'and the file we hold is untouched')
74})