OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-store commits / b76cea2

OverlayRepoStore: records-as-files RepoStore backend (Phase 1)
A drop-in RepoStore backend that stores records as canonical DAG-CBOR files (records/<collection>/<rkey>.dagcbor), overwrite-in-place / unlink-on-delete, so on-disk size is O(current records) not O(operations-ever). MST/commit/CAR are derived on read by the existing atproto-repo codec (read-view consumes only the RepoStore interface). Determinism proof (test, 5/5 · 19/19): byte-identical commit CID, MST root, and CAR vs the Hyperbee RepoStore; records() key-order + getRaw bytes match; delete yields the same MST root as an equivalent fresh repo; head persists across reopen (rev restored, not re-minted); update overwrites in place. Optional hyper-overlay wiring (opts.overlay) for bytes-on-disk dedup/replication; absent = local file tree. See hyper-proto-peer/docs/SPEC-atproto-overlay-storage.md. Tracks hyper-social #8. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
b76cea2b161aa610e4c1a0d71e3a6a18f37fe3a2
by plankatron <fleeky@gmail.com> · 2026-07-19T14:51:09.000Z

Changed files (2) +226 -0

lib/overlay-repo-store.js +126 -0
……@@ -0,0 +1,126 @@
1// OverlayRepoStore — a RepoStore backend that keeps records as canonical DAG-CBOR FILES
2// on disk (one file per record at records/<collection>/<rkey>.dagcbor) instead of in an
3// append-only Hyperbee. Overwrite-in-place on edit, unlink on delete → the on-disk size is
4// O(CURRENT records), not O(operations-ever). MST/commit/CAR are DERIVED on read by the
5// atproto-repo codec (read-view.js / repo-manager.js consume only this interface), so nothing
6// atproto-specific is stored beyond the record bytes + a tiny head.json.
7//
8// Byte-exactness (the determinism contract): `put` stores dagCbor.encode(record) VERBATIM —
9// byte-identical to the Hyperbee RepoStore — so cidForBytesSync(bytes) and the derived MST +
10// commit are identical. records() returns records in Hyperbee key order (utf-8 byte order via
11// b4a.compare) so the derivation is order-stable.
12//
13// The canonical bytes ARE the files on disk. An optional hyper-overlay instance (opts.overlay)
14// indexes + replicates them (bytes stay on disk — no corestore balloon; hyper-overlay/index.js).
15// Without it the store is a fully-functional local file tree (used by the determinism tests).
16// Drop-in for atproto-store/lib/repo-store.js's RepoStore (same method surface).
17
18import * as dagCbor from '@ipld/dag-cbor'
19import b4a from 'b4a'
20import fs from 'node:fs'
21import { join } from 'node:path'
22
23const EXT = '.dagcbor'
24const byteOrder = (a, b) => b4a.compare(b4a.from(a), b4a.from(b)) // match Hyperbee's key comparison exactly
25// atproto collection NSIDs + rkeys are single path segments (no '/'); reject anything else so a
26// record key can never escape the records/ dir or collide with a parent path.
27const isSafeSeg = (s) => typeof s === 'string' && s.length > 0 && !s.includes('/') && s !== '.' && s !== '..'
28
29export class OverlayRepoStore {
30 // root: the per-DID dir (<storage>/dids/<did>). opts.overlay: optional HyperOverlay to
31 // register files with (bytes-on-disk indexing/replication). opts.did: for logging/namespacing.
32 constructor (root, opts = {}) {
33 if (typeof root !== 'string') throw new Error('OverlayRepoStore: root dir path required')
34 this._root = root
35 this._recordsDir = join(root, 'records')
36 this._headFile = join(root, 'head.json')
37 this._overlay = opts.overlay || null
38 this._did = opts.did || null
39 }
40
41 static key (collection, rkey) { return `${collection}/${rkey}` }
42
43 async ready () {
44 fs.mkdirSync(this._recordsDir, { recursive: true })
45 return this
46 }
47
48 _pathFor (collection, rkey) {
49 if (!isSafeSeg(collection) || !isSafeSeg(rkey)) throw new Error(`OverlayRepoStore: unsafe record key "${collection}/${rkey}"`)
50 return join(this._recordsDir, collection, rkey + EXT)
51 }
52
53 async put (collection, rkey, record) {
54 const bytes = b4a.from(dagCbor.encode(record))
55 const p = this._pathFor(collection, rkey)
56 fs.mkdirSync(join(this._recordsDir, collection), { recursive: true })
57 fs.writeFileSync(p, bytes)
58 if (this._overlay) { try { await this._overlay.registerFile(this._overlayPath(collection, rkey), p, { size: bytes.length }) } catch {} }
59 }
60
61 async get (collection, rkey) {
62 const raw = await this.getRaw(collection, rkey)
63 return raw ? raw.record : null
64 }
65
66 // Raw stored bytes + decoded record. The bridge CIDs the EXACT bytes (never a re-encode) —
67 // so these must be the same bytes `put` wrote.
68 async getRaw (collection, rkey) {
69 let bytes
70 try { bytes = fs.readFileSync(this._pathFor(collection, rkey)) } catch (e) { if (e && e.code === 'ENOENT') return null; throw e }
71 return { bytes, record: dagCbor.decode(bytes) }
72 }
73
74 async del (collection, rkey) {
75 try { fs.unlinkSync(this._pathFor(collection, rkey)) } catch (e) { if (!e || e.code !== 'ENOENT') throw e }
76 // The file IS the source of truth; overlay index prune is a Phase-2 concern (evictContent).
77 try { fs.rmdirSync(join(this._recordsDir, collection)) } catch {} // best-effort: drop an empty collection dir
78 }
79
80 // Yield { rkey, record } for one collection, in key (byte) order.
81 async * list (collection) {
82 const dir = join(this._recordsDir, collection)
83 let names
84 try { names = fs.readdirSync(dir) } catch (e) { if (e && e.code === 'ENOENT') return; throw e }
85 for (const n of names.filter((x) => x.endsWith(EXT)).sort(byteOrder)) {
86 const bytes = fs.readFileSync(join(dir, n))
87 yield { rkey: n.slice(0, -EXT.length), record: dagCbor.decode(bytes) }
88 }
89 }
90
91 // All records as { key, bytes, record } in key (byte) order — the sole input to the MST build.
92 async records () {
93 const out = []
94 let collections
95 try { collections = fs.readdirSync(this._recordsDir) } catch (e) { if (e && e.code === 'ENOENT') return out; throw e }
96 for (const collection of collections) {
97 const dir = join(this._recordsDir, collection)
98 let st
99 try { st = fs.statSync(dir) } catch { continue }
100 if (!st.isDirectory()) continue
101 for (const n of fs.readdirSync(dir)) {
102 if (!n.endsWith(EXT)) continue
103 const bytes = fs.readFileSync(join(dir, n))
104 out.push({ key: `${collection}/${n.slice(0, -EXT.length)}`, bytes, record: dagCbor.decode(bytes) })
105 }
106 }
107 out.sort((a, b) => byteOrder(a.key, b.key))
108 return out
109 }
110
111 // Persist the commit head ({ rev, commitCid, dataCid }) — a tiny file, NOT a record, so it never
112 // appears in records()/the MST. This is what keeps the head stable across restart: the rev is
113 // RESTORED, not re-minted (re-minting forges a phantom commit — see RepoManager.init).
114 async saveHead (head) {
115 fs.mkdirSync(this._root, { recursive: true })
116 fs.writeFileSync(this._headFile, JSON.stringify(head))
117 }
118
119 async loadHead () {
120 try { return JSON.parse(fs.readFileSync(this._headFile, 'utf8')) } catch (e) { if (e && e.code === 'ENOENT') return null; throw e }
121 }
122
123 async close () {}
124
125 _overlayPath (collection, rkey) { return `/records/${collection}/${rkey}${EXT}` }
126}
test/overlay-repo-store.test.js +100 -0
……@@ -0,0 +1,100 @@
1// Phase 1 determinism proof: the file-based OverlayRepoStore must derive a BYTE-IDENTICAL
2// atproto repo (commit CID, MST root, CAR) to the append-only Hyperbee RepoStore, so it is a
3// safe drop-in. If these pass, "records as deduped files + MST/commit derived on read" is proven
4// equivalent to today's store — with O(current records) disk instead of O(operations-ever).
5
6import test from 'brittle'
7import { mkdtempSync, rmSync } from 'node:fs'
8import { join } from 'node:path'
9import { tmpdir } from 'node:os'
10import b4a from 'b4a'
11import { RepoStore } from '../lib/repo-store.js'
12import { OverlayRepoStore } from '../lib/overlay-repo-store.js'
13import { getRepo } from '../lib/read-view.js'
14
15const DID = 'did:plc:test123'
16const KEY = (() => { const k = b4a.alloc(32); k[31] = 7; return k })() // fixed valid secp256k1 scalar (deterministic sig)
17const REV = '3ktestrev00000'
18const META = { did: DID, rev: REV, prev: null, privateKey: KEY }
19
20const RECORDS = [
21 { collection: 'app.bsky.actor.profile', rkey: 'self', record: { $type: 'app.bsky.actor.profile', displayName: 'Test' } },
22 { collection: 'app.bsky.feed.post', rkey: '3kaaa', record: { $type: 'app.bsky.feed.post', text: 'hello', createdAt: '2026-01-01T00:00:00.000Z', langs: ['en'] } },
23 { collection: 'app.bsky.feed.post', rkey: '3kbbb', record: { $type: 'app.bsky.feed.post', text: 'world', createdAt: '2026-01-02T00:00:00.000Z', langs: ['en'] } },
24 { collection: 'app.bsky.graph.follow', rkey: '3kccc', record: { $type: 'app.bsky.graph.follow', subject: 'did:plc:other', createdAt: '2026-01-03T00:00:00.000Z' } }
25]
26
27async function fill (store) { for (const r of RECORDS) await store.put(r.collection, r.rkey, r.record); return store }
28function tmp (t) { const d = mkdtempSync(join(tmpdir(), 'ors-')); t.teardown(() => rmSync(d, { recursive: true, force: true })); return d }
29
30test('OverlayRepoStore derives byte-identical commit + MST + CAR as the Hyperbee RepoStore', async (t) => {
31 const hb = await fill(await new RepoStore(tmp(t)).ready())
32 const ov = await fill(await new OverlayRepoStore(tmp(t)).ready())
33
34 const a = await getRepo(hb, META)
35 const b = await getRepo(ov, META)
36
37 t.is(b.commitCid.toString(), a.commitCid.toString(), 'commit CID identical')
38 t.is(b.dataCid.toString(), a.dataCid.toString(), 'MST root (data CID) identical')
39 t.ok(b4a.equals(b.carBytes, a.carBytes), 'CAR bytes byte-identical')
40
41 await hb.close(); await ov.close()
42})
43
44test('records() order + getRaw bytes match Hyperbee exactly', async (t) => {
45 const hb = await fill(await new RepoStore(tmp(t)).ready())
46 const ov = await fill(await new OverlayRepoStore(tmp(t)).ready())
47
48 const ra = await hb.records(); const rb = await ov.records()
49 t.is(rb.length, ra.length, 'same record count')
50 t.alike(rb.map((r) => r.key), ra.map((r) => r.key), 'same keys in same (byte) order')
51 for (let i = 0; i < ra.length; i++) t.ok(b4a.equals(rb[i].bytes, ra[i].bytes), 'record bytes identical: ' + ra[i].key)
52
53 const ga = await hb.getRaw('app.bsky.feed.post', '3kaaa')
54 const gb = await ov.getRaw('app.bsky.feed.post', '3kaaa')
55 t.ok(b4a.equals(gb.bytes, ga.bytes), 'getRaw bytes identical')
56 t.alike(gb.record, ga.record, 'getRaw decoded record identical')
57
58 await hb.close(); await ov.close()
59})
60
61test('delete removes the file and shifts the derived MST (O(current state), matches a fresh repo)', async (t) => {
62 const ov = await fill(await new OverlayRepoStore(tmp(t)).ready())
63 const before = await getRepo(ov, META)
64 await ov.del('app.bsky.feed.post', '3kbbb')
65 t.is(await ov.get('app.bsky.feed.post', '3kbbb'), null, 'record gone after delete')
66 const after = await getRepo(ov, META)
67 t.not(after.dataCid.toString(), before.dataCid.toString(), 'MST root changed after delete')
68
69 // an equivalent FRESH repo (built without the deleted record) derives the SAME root — proof the
70 // post-delete state is O(current), not "old record still lingering in an append-only log".
71 const hb = await new RepoStore(tmp(t)).ready()
72 for (const r of RECORDS) if (r.rkey !== '3kbbb') await hb.put(r.collection, r.rkey, r.record)
73 const fresh = await getRepo(hb, META)
74 t.is(after.dataCid.toString(), fresh.dataCid.toString(), 'post-delete MST root == equivalent fresh repo')
75
76 await ov.close(); await hb.close()
77})
78
79test('head persists across reopen (rev restored, not re-minted) + records survive', async (t) => {
80 const dir = tmp(t)
81 const ov = await fill(await new OverlayRepoStore(dir).ready())
82 await ov.saveHead({ rev: REV, commitCid: 'bafyCommit', dataCid: 'bafyData' })
83 await ov.close()
84
85 const re = await new OverlayRepoStore(dir).ready()
86 const h = await re.loadHead()
87 t.is(h.rev, REV, 'rev restored from head.json')
88 t.is(h.commitCid, 'bafyCommit', 'commitCid restored')
89 t.is((await re.records()).length, RECORDS.length, 'records survive reopen')
90 await re.close()
91})
92
93test('update overwrites in place (one file, not an append) and re-derives correctly', async (t) => {
94 const ov = await fill(await new OverlayRepoStore(tmp(t)).ready())
95 await ov.put('app.bsky.feed.post', '3kaaa', { $type: 'app.bsky.feed.post', text: 'edited', createdAt: '2026-01-01T00:00:00.000Z', langs: ['en'] })
96 const got = await ov.get('app.bsky.feed.post', '3kaaa')
97 t.is(got.text, 'edited', 'record overwritten in place')
98 t.is((await ov.records()).length, RECORDS.length, 'still exactly one file per record (no append)')
99 await ov.close()
100})